227 lines
12 KiB
Python
227 lines
12 KiB
Python
"""Content questions must get source passages, not a metadata-only substitute."""
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
|
from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
QUESTION = 'Čomu sa venuje diplomová práca Jána Holpa? Odpovedz podľa dostupných podkladov a uveď zdroj.'
|
|
ASSIGNMENT = ('1. Vypracujte prehľad metód ohodnotenia dokumentov v systémoch získavania informácií.\n'
|
|
'2. Implementujte systém pre získavanie informácií v slovenskom jazyku.')
|
|
URL = 'https://example.test/jan'
|
|
|
|
|
|
@pytest.fixture
|
|
def payload():
|
|
sources = [
|
|
{'source_id': 'S1', 'title': 'Ján Holp', 'author': 'Daniel Hladek',
|
|
'document_path': 'pages/students/2016/jan_holp/README.md', 'source_url': URL,
|
|
'text': 'Dokument: Ján Holp\nSekcia: Diplomová práca 2021\n'
|
|
'Názov: Systém získavania informácií v slovenskom jazyku.\n'
|
|
'Meno vedúceho: Ing. Daniel Hladek, PhD.\nZadanie diplomovej práce:\n' + ASSIGNMENT +
|
|
'\n## Metódy\nVyhodnotenie využíva PageRank.\n'
|
|
'## Bakalárska práca 2019\nNázov: Vyhľadávač na slovenskom internete.\n'
|
|
'Práca implementuje prehľadávanie slovenského webu.\n'
|
|
'## Diplomový projekt 2020\nNázov: Pokusy s databázou.\n'
|
|
'Projekt využíva databázu Cassandra.'},
|
|
{'source_id': 'S2', 'title': 'Boris Nový',
|
|
'document_path': 'pages/students/2020/boris_novy/README.md',
|
|
'source_url': 'https://example.test/boris',
|
|
'text': 'Sekcia: Diplomová práca 2021\nNázov: Spracovanie obrázkov.\n'
|
|
'Práca skúma klasifikáciu medicínskych obrázkov.'}]
|
|
return {'status': 'ok', 'reason': None, 'sources': sources, 'context': build_context_text(sources),
|
|
'complete': False, 'truncated': True, 'truncation_reasons': ['result_limit'],
|
|
'index_version': {'version': 'fixture'}}
|
|
|
|
|
|
def description():
|
|
return {'answer': '', 'status': 'ok', 'claims': [], 'count': None,
|
|
'source_urls': [URL], 'evidence_quotes': [{'source_id': 'S1', 'quote': ASSIGNMENT}]}
|
|
|
|
|
|
def metadata():
|
|
return {'answer': '', 'status': 'ok', 'claims': [{'person': 'Ján Holp', 'work_type': 'diploma_thesis',
|
|
'title': 'Systém získavania informácií v slovenskom jazyku', 'year': 2021}],
|
|
'evidence_quotes': [], 'source_urls': [URL]}
|
|
|
|
|
|
def completion(answer):
|
|
return {'choices': [{'message': {'content': json.dumps(answer, ensure_ascii=False)}}]}
|
|
|
|
|
|
@pytest.mark.parametrize('question', [QUESTION,
|
|
'O čom je diplomová práca Jána Holpa?',
|
|
'Čomu sa venovala diplomová práca Jána Holpa?',
|
|
'Čo rieši diplomová práca Jána Holpa?',
|
|
'Stručne opíš diplomovú prácu Jána Holpa.',
|
|
'Zhrň obsah diplomovej práce Jána Holpa.',
|
|
'Aké sú ciele diplomovej práce Jána Holpa?',
|
|
'Aké je zadanie diplomovej práce Jána Holpa?',
|
|
'Aké metódy používa diplomová práca Jána Holpa?'])
|
|
def test_content_intent_is_not_a_title_or_year_lookup(payload, question):
|
|
c = rag_evidence_context(question, payload)
|
|
assert c['requirements']['content']
|
|
assert c['requirements']['response_mode'] == 'quotes'
|
|
assert not c['requirements']['title'] and not c['requirements']['year']
|
|
assert any(ASSIGNMENT in p['text'] for p in c['content_passages'])
|
|
|
|
|
|
def test_named_work_content_without_explicit_work_type_still_needs_text(payload):
|
|
c = rag_evidence_context('Opíš obsah práce Jána Holpa.', payload)
|
|
assert c['requirements']['content'] and c['requirements']['response_mode'] == 'quotes'
|
|
assert 'missing_content_evidence' in rag_answer_issues(metadata(), c)
|
|
|
|
|
|
@pytest.mark.parametrize('question', [
|
|
'Aký je názov diplomovej práce Jána Holpa?',
|
|
'Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?',
|
|
'Uveď názov a rok diplomovej práce Jána Holpa.',
|
|
'V ktorom roku je diplomová práca Jána Holpa?'])
|
|
def test_identity_questions_keep_the_existing_typed_binding(payload, question):
|
|
c = rag_evidence_context(question, payload)
|
|
assert c['requirements']['response_mode'] == 'claims'
|
|
assert not c['requirements']['content']
|
|
assert rag_answer_issues(metadata(), c) == []
|
|
bad = metadata(); bad['claims'][0]['year'] = 2019
|
|
assert 'unsupported_person_work_type_year_binding' in rag_answer_issues(bad, c)
|
|
|
|
|
|
@pytest.mark.parametrize('question', [
|
|
'Aký je názov diplomovej práce Jána Holpa, ktorá používa PageRank?',
|
|
'Aký je názov diplomovej práce Jána Holpa o metódach vyhľadávania?',
|
|
'Uveď názov diplomovej práce Jána Holpa „Metódy a obsah implementácie“.'])
|
|
def test_topic_words_in_identity_question_do_not_request_a_description(payload, question):
|
|
c = rag_evidence_context(question, payload)
|
|
assert not c['requirements']['content'] and c['requirements']['response_mode'] == 'claims'
|
|
|
|
|
|
def test_passages_exclude_other_people_and_other_work_types(payload):
|
|
passages = rag_evidence_context(QUESTION, payload)['content_passages']
|
|
assert passages and all(p['source_id'] == 'S1' and p['work_type'] == 'diploma_thesis' for p in passages)
|
|
assert all('Cassandra' not in p['text'] and 'medicínskych' not in p['text'] for p in passages)
|
|
|
|
|
|
@pytest.mark.parametrize('year,expected', [(2021, True), (1999, False)])
|
|
def test_year_constraint_also_applies_to_content(payload, year, expected):
|
|
c = rag_evidence_context(f'Opíš obsah diplomovej práce Jána Holpa v roku {year}.', payload)
|
|
assert bool(c['content_passages']) == expected
|
|
assert all(p['year'] == year for p in c['content_passages'])
|
|
if not expected:
|
|
assert 'quote_outside_requested_work' in rag_answer_issues(description(), c)
|
|
refusal = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
|
assert rag_answer_issues(refusal, c) == []
|
|
|
|
|
|
@pytest.mark.parametrize('quote', ['Názov: Systém získavania informácií v slovenskom jazyku.',
|
|
'Systém získavania informácií v slovenskom jazyku.',
|
|
'Meno vedúceho: Ing. Daniel Hladek, PhD.', 'Zadanie diplomovej práce:'])
|
|
def test_metadata_or_an_assignment_heading_does_not_describe_work(payload, quote):
|
|
c = rag_evidence_context(QUESTION, payload)
|
|
answer = description(); answer['evidence_quotes'][0]['quote'] = quote
|
|
assert 'metadata_quote_does_not_describe_work' in rag_answer_issues(answer, c)
|
|
|
|
|
|
def test_metadata_claims_alone_are_rejected(payload):
|
|
issues = rag_answer_issues(metadata(), rag_evidence_context(QUESTION, payload))
|
|
assert 'missing_content_evidence' in issues and 'unrequested_structured_claims' in issues
|
|
|
|
|
|
@pytest.mark.parametrize('source_id,quote', [
|
|
('S1', 'Práca implementuje prehľadávanie slovenského webu.'),
|
|
('S1', 'Projekt využíva databázu Cassandra.'),
|
|
('S2', 'Práca skúma klasifikáciu medicínskych obrázkov.')])
|
|
def test_literal_quote_from_wrong_work_or_person_is_rejected(payload, source_id, quote):
|
|
answer = description(); answer['source_urls'] += ['https://example.test/boris']
|
|
answer['evidence_quotes'] = [{'source_id': source_id, 'quote': quote}]
|
|
assert 'quote_outside_requested_work' in rag_answer_issues(answer, rag_evidence_context(QUESTION, payload))
|
|
|
|
|
|
def test_correction_sends_real_description_but_no_unrelated_sections(payload):
|
|
payload = compact_rag_payload(payload)
|
|
outputs = iter([metadata(), description()]); calls = []
|
|
def complete(messages):
|
|
calls.append(copy.deepcopy(messages))
|
|
return completion(next(outputs))
|
|
result = generate_verified_rag(QUESTION, payload, complete)
|
|
assert result['accepted_for_delivery'] and len(calls) == 2
|
|
sent = json.loads(calls[0][1]['content'])['context']
|
|
assert 'facts' not in sent and 'context' not in sent
|
|
text = json.dumps(sent['content_candidates'], ensure_ascii=False)
|
|
assert all(c.get('evidence_id') for c in sent['content_candidates'])
|
|
assert all('text' not in s for s in sent['sources'])
|
|
assert 'ohodnotenia dokumentov' in text and 'Cassandra' not in text and 'medicínskych' not in text
|
|
assert 'missing_content_evidence' in result['attempts'][0]['issues']
|
|
assert 'Vypracovanie prehľadu metód ohodnotenia dokumentov' in result['answer']
|
|
assert 'Implementácia systému pre získavanie informácií v slovenskom jazyku.' in result['answer']
|
|
assert result['attempts'][-1]['parsed_answer']['evidence_quotes'][0]['quote'] == ASSIGNMENT
|
|
assert 'Výpis je neúplný.' not in result['answer']
|
|
assert result['complete'] is False and result['answer_coverage']['status'] == 'supported'
|
|
assert 'expected_bindings' not in json.dumps(calls)
|
|
|
|
|
|
def test_two_metadata_only_attempts_never_reach_user(payload):
|
|
result = generate_verified_rag(QUESTION, payload, lambda _: completion(metadata()))
|
|
assert not result['accepted_for_delivery'] and len(result['attempts']) == 2
|
|
|
|
|
|
def test_name_year_and_description_can_be_requested_together(payload):
|
|
question = 'Uveď názov, rok a stručne opíš obsah diplomovej práce Jána Holpa.'
|
|
c = rag_evidence_context(question, payload)
|
|
assert c['requirements']['response_mode'] == 'mixed'
|
|
good = {**metadata(), 'evidence_quotes': description()['evidence_quotes']}
|
|
assert rag_answer_issues(good, c) == []
|
|
assert 'missing_content_evidence' in rag_answer_issues(metadata(), c)
|
|
assert 'missing_requested_title' in rag_answer_issues(description(), c)
|
|
|
|
|
|
def test_overview_with_documented_assignment_cannot_be_falsely_refused(payload):
|
|
answer = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
|
assert 'rejected_supported_work_description' in rag_answer_issues(answer, rag_evidence_context(QUESTION, payload))
|
|
|
|
|
|
def test_no_description_in_delivered_text_does_not_license_an_invented_one(payload):
|
|
payload['sources'] = [payload['sources'][0]]
|
|
payload['sources'][0]['text'] = 'Sekcia: Diplomová práca 2021\nNázov: Systém získavania informácií v slovenskom jazyku.'
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
c = rag_evidence_context(QUESTION, payload)
|
|
assert c['content_passages'] == []
|
|
answer = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
|
assert rag_answer_issues(answer, c) == []
|
|
|
|
|
|
def test_a_request_for_results_is_not_answered_merely_by_a_proposed_assignment(payload):
|
|
c = rag_evidence_context('Zhrň dosiahnuté výsledky diplomovej práce Jána Holpa.', payload)
|
|
answer = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
|
assert rag_answer_issues(answer, c) == []
|
|
|
|
|
|
def test_same_person_other_diploma_year_does_not_satisfy_content_constraint(payload):
|
|
payload['sources'][0]['text'] += '\n## Diplomová práca 2022\nNázov: Iná diplomová téma.\nPráca skúma automatické preklady.'
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
question = 'Uveď názov a rok a opíš diplomovú prácu Jána Holpa v roku 2021.'
|
|
c = rag_evidence_context(question, payload)
|
|
answer = metadata(); answer['evidence_quotes'] = [{'source_id': 'S1', 'quote': 'Práca skúma automatické preklady.'}]
|
|
assert 'quote_outside_requested_work' in rag_answer_issues(answer, c)
|
|
assert 'missing_bound_content_evidence' in rag_answer_issues(answer, c)
|
|
|
|
|
|
def test_mixed_answer_cannot_attach_another_years_description_to_its_claim(payload):
|
|
payload['sources'][0]['text'] += '\n## Diplomová práca 2022\nNázov: Iná diplomová téma.\nPráca skúma automatické preklady.'
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
c = rag_evidence_context('Uveď názov, rok a opíš diplomovú prácu Jána Holpa.', payload)
|
|
answer = metadata(); answer['evidence_quotes'] = [{'source_id': 'S1', 'quote': 'Práca skúma automatické preklady.'}]
|
|
assert 'missing_bound_content_evidence' in rag_answer_issues(answer, c)
|
|
|
|
|
|
def test_comparing_contents_requires_evidence_for_each_requested_person(payload):
|
|
c = rag_evidence_context('Porovnaj obsah diplomových prác osôb Ján Holp a Boris Nový.', payload)
|
|
answer = description()
|
|
assert 'missing_bound_content_evidence' in rag_answer_issues(answer, c)
|
|
answer['evidence_quotes'].append({'source_id': 'S2', 'quote': 'Práca skúma klasifikáciu medicínskych obrázkov.'})
|
|
answer['source_urls'].append('https://example.test/boris')
|
|
assert rag_answer_issues(answer, c) == []
|