203 lines
11 KiB
Python
203 lines
11 KiB
Python
"""Technical retrieval limits must not become false claims about answer quality."""
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.answer_evaluation.answer_coverage import answer_coverage
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
|
from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues, render_rag_answer
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
|
|
URL = 'https://example.test/eva'
|
|
GOALS = '1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.\n3. Vyhodnoťte presnosť.\n4. Navrhnite zlepšenia.'
|
|
|
|
|
|
@pytest.fixture
|
|
def payload():
|
|
sources = [{'source_id': 'S1', 'title': 'Eva Nová', 'document_path': 'pages/students/2020/eva_nova/README.md',
|
|
'source_url': URL, 'text': 'Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\nZadanie:\n' + GOALS}]
|
|
return {'status': 'ok', 'reason': None, 'complete': False, 'truncated': True,
|
|
'truncation_reasons': ['candidate_limit', 'per_document_limit', 'result_limit'],
|
|
'index_version': {'version': 'fixture'}, 'sources': sources, 'context': build_context_text(sources)}
|
|
|
|
|
|
def answer():
|
|
return {'status': 'ok', 'answer': '', 'claims': [{'person': 'Eva Nová', 'work_type': 'diploma_thesis',
|
|
'title': 'Vyhľadávanie informácií', 'year': 2026}], 'evidence_quotes': [], 'source_urls': [URL]}
|
|
|
|
|
|
def quote(text):
|
|
return {'status': 'ok', 'answer': '', 'claims': [], 'source_urls': [URL],
|
|
'evidence_quotes': [{'source_id': 'S1', 'quote': text}]}
|
|
|
|
|
|
@pytest.mark.parametrize('complete,truncated,reasons', [
|
|
(False, True, ['candidate_limit', 'per_document_limit', 'result_limit']),
|
|
(None, False, []), (False, True, ['llm_context_budget'])])
|
|
def test_supported_fact_is_not_an_incomplete_answer(payload, complete, truncated, reasons):
|
|
payload.update(complete=complete, truncated=truncated, truncation_reasons=reasons)
|
|
c = rag_evidence_context('Aký je názov a rok diplomovej práce Evy Novej?', payload)
|
|
before = copy.deepcopy(c)
|
|
coverage = answer_coverage(answer(), c)
|
|
assert coverage['status'] == 'supported' and coverage['warnings'] == []
|
|
text = render_rag_answer(answer(), c)
|
|
assert '2026' in text and URL in text and 'neúpln' not in text
|
|
assert c == before # Never fake complete=true or remove diagnostic limits.
|
|
|
|
|
|
@pytest.mark.parametrize('question', ['Vypíš všetky práce Evy Novej.', 'Aké práce má Eva Nová?',
|
|
'Uveď kompletný zoznam prác Evy Novej.',
|
|
'Vypíš všetky bakalárske a diplomové práce Evy Novej.'])
|
|
def test_limited_record_list_still_warns(payload, question):
|
|
c = rag_evidence_context(question, payload)
|
|
coverage = answer_coverage(answer(), c)
|
|
assert coverage['scope'] == 'list' and coverage['status'] == 'partial'
|
|
assert 'Uvedený zoznam nemusí obsahovať všetky záznamy.' in render_rag_answer(answer(), c)
|
|
|
|
|
|
def test_explicitly_complete_retrieval_does_not_add_a_list_warning(payload):
|
|
payload.update(complete=True, truncated=False, truncation_reasons=[])
|
|
c = rag_evidence_context('Vypíš všetky práce Evy Novej.', payload)
|
|
assert answer_coverage(answer(), c)['warnings'] == []
|
|
|
|
|
|
def test_quoted_title_is_not_an_exhaustive_list_request(payload):
|
|
c = rag_evidence_context('Aký je rok diplomovej práce „Všetky práce v katalógu“?', payload)
|
|
assert answer_coverage(answer(), c)['scope'] == 'fact'
|
|
|
|
|
|
def test_unknown_list_coverage_is_not_silently_exhaustive(payload):
|
|
payload.update(complete=None, truncated=False, truncation_reasons=[])
|
|
c = rag_evidence_context('Aké práce má Eva Nová?', payload)
|
|
assert answer_coverage(answer(), c)['warnings'] == ['list_may_be_incomplete']
|
|
|
|
|
|
def test_legacy_context_without_limit_metadata_remains_usable(payload):
|
|
payload.pop('truncation_reasons'); payload.pop('complete'); payload.pop('truncated')
|
|
c = rag_evidence_context('Aký je názov diplomovej práce Evy Novej?', payload)
|
|
assert answer_coverage(answer(), c)['warnings'] == []
|
|
assert 'Vyhľadávanie informácií' in render_rag_answer(answer(), c)
|
|
|
|
|
|
def test_regular_assignment_can_show_all_supplied_goals_without_global_warning(payload):
|
|
c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload)
|
|
a = quote(GOALS)
|
|
assert rag_answer_issues(a, c) == []
|
|
text = render_rag_answer(a, c)
|
|
assert '1. Preskúmanie metód vyhľadávania.' in text
|
|
assert '4. Návrh zlepšení.' in text
|
|
assert 'neúpln' not in text and 'Zo zdroja' not in text
|
|
assert answer_coverage(a, c)['scope'] == 'assignment'
|
|
|
|
|
|
def test_clipped_assignment_gets_specific_warning(payload):
|
|
payload['sources'][0]['text'] += '\n[OBSAH ZDROJA OREZANÝ]'
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload)
|
|
a = quote('1. Preskúmajte metódy vyhľadávania.')
|
|
coverage = answer_coverage(a, c)
|
|
assert coverage['status'] == 'partial' and coverage['warnings'] == ['assignment_text_clipped']
|
|
assert 'uvedené body nemusia tvoriť celé zadanie' in render_rag_answer(a, c)
|
|
|
|
|
|
def test_request_for_all_assignment_items_preserves_uncertainty(payload):
|
|
c = rag_evidence_context('Uveď všetky body zadania diplomovej práce Evy Novej.', payload)
|
|
assert answer_coverage(quote(GOALS), c)['warnings'] == ['assignment_completeness_unknown']
|
|
|
|
|
|
@pytest.mark.parametrize('budget', [False, True])
|
|
def test_no_answer_is_bounded_and_only_reports_actual_context_cut(payload, budget):
|
|
payload['truncation_reasons'] = ['llm_context_budget'] if budget else ['result_limit']
|
|
c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej v roku 1999?', payload)
|
|
a = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
|
text = render_rag_answer(a, c)
|
|
assert 'Eva Nová' in text and 'v dostupných podkladoch sa nenašiel záznam diplomovej práce z roku **1999**' in text
|
|
assert 'Výpis je neúplný' not in text
|
|
assert ('Časť podkladov bola pri príprave kontextu skrátená.' in text) == budget
|
|
assert answer_coverage(a, c)['status'] == 'not_found'
|
|
|
|
|
|
def test_irrelevant_dropped_source_does_not_make_cited_assignment_partial(payload):
|
|
payload['sources'].append({'source_id': 'S2', 'title': 'Iný dokument', 'source_url': 'https://example.test/other',
|
|
'document_path': 'pages/topics/other/README.md', 'text': 'Iný obsah\n[OBSAH ZDROJA OREZANÝ]'})
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
payload['truncation_reasons'].append('llm_context_budget')
|
|
c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload)
|
|
assert answer_coverage(quote(GOALS), c)['warnings'] == []
|
|
|
|
|
|
def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload):
|
|
c = rag_evidence_context('Koľko prác je evidovaných?', payload)
|
|
c['aggregation'] = {'count': 2, 'complete': True}
|
|
a = {'status': 'ok', 'claims': [], 'evidence_quotes': [], 'source_urls': [], 'count': 2}
|
|
assert answer_coverage(a, c)['status'] == 'supported'
|
|
c['aggregation']['complete'] = False
|
|
assert 'count_not_established' in answer_coverage(a, c)['warnings']
|
|
|
|
|
|
def test_overview_preserves_all_main_assignment_items_without_another_model_request(payload):
|
|
question = 'Čomu sa venuje diplomová práca Evy Novej?'
|
|
short = '1. Preskúmanie metód vyhľadávania.\n2. Implementácia vyhľadávača.'
|
|
c = rag_evidence_context(question, payload)
|
|
assert rag_answer_issues(quote(GOALS), c) == []
|
|
calls = []
|
|
def complete(messages):
|
|
calls.append(messages)
|
|
return {'choices': [{'message': {'content': json.dumps(quote(GOALS))}}]}
|
|
result = generate_verified_rag(question, payload, complete)
|
|
assert result['accepted_for_delivery'] and len(result['attempts']) == len(calls) == 1
|
|
assert short in result['answer'] and '3. Vyhodnotenie presnosti.' in result['answer']
|
|
assert '4. Návrh zlepšení.' in result['answer']
|
|
assert result['attempts'][0]['parsed_answer']['evidence_quotes'][0]['quote'] == GOALS
|
|
assert result['complete'] is False and result['truncation_reasons'] == payload['truncation_reasons']
|
|
assert result['answer_coverage']['status'] == 'supported' and result['answer_coverage']['warnings'] == []
|
|
|
|
|
|
def test_short_comparison_can_cover_three_people_instead_of_dropping_one(payload):
|
|
sources = []
|
|
quotes = []
|
|
for index, (name, slug) in enumerate([('Eva Nová', 'eva_nova'), ('Boris Nový', 'boris_novy'),
|
|
('Cyril Malý', 'cyril_maly')], 1):
|
|
source = {**payload['sources'][0], 'source_id': f'S{index}', 'title': name,
|
|
'source_url': f'https://example.test/{slug}',
|
|
'document_path': f'pages/students/2020/{slug}/README.md'}
|
|
sources.append(source)
|
|
quotes.append({'source_id': source['source_id'], 'quote': GOALS})
|
|
payload['sources'] = sources; payload['context'] = build_context_text(sources)
|
|
c = rag_evidence_context('Porovnaj obsah diplomových prác osôb Eva Nová, Boris Nový a Cyril Malý.', payload)
|
|
assert len(c['requirements']['people']) == 3
|
|
a = {'status': 'ok', 'claims': [], 'evidence_quotes': quotes, 'source_urls': [s['source_url'] for s in sources]}
|
|
assert rag_answer_issues(a, c) == []
|
|
a['evidence_quotes'].pop()
|
|
assert 'missing_bound_content_evidence' in rag_answer_issues(a, c)
|
|
|
|
|
|
def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidence(payload):
|
|
from tools.answer_evaluation.verified_rag import overview_excerpts
|
|
extra = 'Stav:\n\n- Implementované indexovanie dokumentov.'
|
|
payload['sources'][0]['text'] += '\n' + extra
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload)
|
|
a = quote(GOALS)
|
|
a['evidence_quotes'].append({'source_id': 'S1', 'quote': extra})
|
|
before = copy.deepcopy(a)
|
|
assert rag_answer_issues(a, c) == []
|
|
units = overview_excerpts(a, c)
|
|
assert len(units) == 4 and all(line in units for line in GOALS.splitlines())
|
|
assert all(any(unit in evidence['quote'] for evidence in a['evidence_quotes']) for unit in units)
|
|
assert a == before
|
|
|
|
|
|
def test_long_prose_excerpt_is_selected_without_inventing_a_summary(payload):
|
|
from tools.answer_evaluation.verified_rag import overview_excerpts
|
|
prose = 'Práca skúma vyhľadávanie v slovenských dokumentoch. ' * 18
|
|
payload['sources'][0]['text'] = 'Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\n'+prose
|
|
payload['context'] = build_context_text(payload['sources'])
|
|
c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload)
|
|
a = quote(prose)
|
|
assert rag_answer_issues(a, c) == []
|
|
units = overview_excerpts(a, c)
|
|
assert units and sum(map(len, units)) <= 600 and all(unit in prose for unit in units)
|