"""Technical retrieval limits must not become false claims about answer quality.""" import copy import json import pytest from tools.answer_evaluation.answer_coverage import answer_coverage from tools.answer_evaluation.rag_evidence import rag_evidence_context from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues, render_rag_answer from tools.rag.rag_utils import build_context_text URL = 'https://example.test/eva' GOALS = '1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.\n3. Vyhodnoťte presnosť.\n4. Navrhnite zlepšenia.' @pytest.fixture def payload(): sources = [{'source_id': 'S1', 'title': 'Eva Nová', 'document_path': 'pages/students/2020/eva_nova/README.md', 'source_url': URL, 'text': 'Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\nZadanie:\n' + GOALS}] return {'status': 'ok', 'reason': None, 'complete': False, 'truncated': True, 'truncation_reasons': ['candidate_limit', 'per_document_limit', 'result_limit'], 'index_version': {'version': 'fixture'}, 'sources': sources, 'context': build_context_text(sources)} def answer(): return {'status': 'ok', 'answer': '', 'claims': [{'person': 'Eva Nová', 'work_type': 'diploma_thesis', 'title': 'Vyhľadávanie informácií', 'year': 2026}], 'evidence_quotes': [], 'source_urls': [URL]} def quote(text): return {'status': 'ok', 'answer': '', 'claims': [], 'source_urls': [URL], 'evidence_quotes': [{'source_id': 'S1', 'quote': text}]} @pytest.mark.parametrize('complete,truncated,reasons', [ (False, True, ['candidate_limit', 'per_document_limit', 'result_limit']), (None, False, []), (False, True, ['llm_context_budget'])]) def test_supported_fact_is_not_an_incomplete_answer(payload, complete, truncated, reasons): payload.update(complete=complete, truncated=truncated, truncation_reasons=reasons) c = rag_evidence_context('Aký je názov a rok diplomovej práce Evy Novej?', payload) before = copy.deepcopy(c) coverage = answer_coverage(answer(), c) assert coverage['status'] == 'supported' and coverage['warnings'] == [] text = render_rag_answer(answer(), c) assert '2026' in text and URL in text and 'neúpln' not in text assert c == before # Never fake complete=true or remove diagnostic limits. @pytest.mark.parametrize('question', ['Vypíš všetky práce Evy Novej.', 'Aké práce má Eva Nová?', 'Uveď kompletný zoznam prác Evy Novej.', 'Vypíš všetky bakalárske a diplomové práce Evy Novej.']) def test_limited_record_list_still_warns(payload, question): c = rag_evidence_context(question, payload) coverage = answer_coverage(answer(), c) assert coverage['scope'] == 'list' and coverage['status'] == 'partial' assert 'Uvedený zoznam nemusí obsahovať všetky záznamy.' in render_rag_answer(answer(), c) def test_explicitly_complete_retrieval_does_not_add_a_list_warning(payload): payload.update(complete=True, truncated=False, truncation_reasons=[]) c = rag_evidence_context('Vypíš všetky práce Evy Novej.', payload) assert answer_coverage(answer(), c)['warnings'] == [] def test_quoted_title_is_not_an_exhaustive_list_request(payload): c = rag_evidence_context('Aký je rok diplomovej práce „Všetky práce v katalógu“?', payload) assert answer_coverage(answer(), c)['scope'] == 'fact' def test_unknown_list_coverage_is_not_silently_exhaustive(payload): payload.update(complete=None, truncated=False, truncation_reasons=[]) c = rag_evidence_context('Aké práce má Eva Nová?', payload) assert answer_coverage(answer(), c)['warnings'] == ['list_may_be_incomplete'] def test_legacy_context_without_limit_metadata_remains_usable(payload): payload.pop('truncation_reasons'); payload.pop('complete'); payload.pop('truncated') c = rag_evidence_context('Aký je názov diplomovej práce Evy Novej?', payload) assert answer_coverage(answer(), c)['warnings'] == [] assert 'Vyhľadávanie informácií' in render_rag_answer(answer(), c) def test_regular_assignment_can_show_all_supplied_goals_without_global_warning(payload): c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload) a = quote(GOALS) assert rag_answer_issues(a, c) == [] text = render_rag_answer(a, c) assert '1. Preskúmanie metód vyhľadávania.' in text assert '4. Návrh zlepšení.' in text assert 'neúpln' not in text and 'Zo zdroja' not in text assert answer_coverage(a, c)['scope'] == 'assignment' def test_clipped_assignment_gets_specific_warning(payload): payload['sources'][0]['text'] += '\n[OBSAH ZDROJA OREZANÝ]' payload['context'] = build_context_text(payload['sources']) c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload) a = quote('1. Preskúmajte metódy vyhľadávania.') coverage = answer_coverage(a, c) assert coverage['status'] == 'partial' and coverage['warnings'] == ['assignment_text_clipped'] assert 'uvedené body nemusia tvoriť celé zadanie' in render_rag_answer(a, c) def test_request_for_all_assignment_items_preserves_uncertainty(payload): c = rag_evidence_context('Uveď všetky body zadania diplomovej práce Evy Novej.', payload) assert answer_coverage(quote(GOALS), c)['warnings'] == ['assignment_completeness_unknown'] @pytest.mark.parametrize('budget', [False, True]) def test_no_answer_is_bounded_and_only_reports_actual_context_cut(payload, budget): payload['truncation_reasons'] = ['llm_context_budget'] if budget else ['result_limit'] c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej v roku 1999?', payload) a = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []} text = render_rag_answer(a, c) assert 'Eva Nová' in text and 'v dostupných podkladoch sa nenašiel záznam diplomovej práce z roku **1999**' in text assert 'Výpis je neúplný' not in text assert ('Časť podkladov bola pri príprave kontextu skrátená.' in text) == budget assert answer_coverage(a, c)['status'] == 'not_found' def test_irrelevant_dropped_source_does_not_make_cited_assignment_partial(payload): payload['sources'].append({'source_id': 'S2', 'title': 'Iný dokument', 'source_url': 'https://example.test/other', 'document_path': 'pages/topics/other/README.md', 'text': 'Iný obsah\n[OBSAH ZDROJA OREZANÝ]'}) payload['context'] = build_context_text(payload['sources']) payload['truncation_reasons'].append('llm_context_budget') c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload) assert answer_coverage(quote(GOALS), c)['warnings'] == [] def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload): c = rag_evidence_context('Koľko prác je evidovaných?', payload) c['aggregation'] = {'count': 2, 'complete': True} a = {'status': 'ok', 'claims': [], 'evidence_quotes': [], 'source_urls': [], 'count': 2} assert answer_coverage(a, c)['status'] == 'supported' c['aggregation']['complete'] = False assert 'count_not_established' in answer_coverage(a, c)['warnings'] def test_long_valid_overview_is_shortened_without_another_model_request(payload): question = 'Čomu sa venuje diplomová práca Evy Novej?' short = '1. Preskúmanie metód vyhľadávania.\n2. Implementácia vyhľadávača.' c = rag_evidence_context(question, payload) assert rag_answer_issues(quote(GOALS), c) == [] calls = [] def complete(messages): calls.append(messages) return {'choices': [{'message': {'content': json.dumps(quote(GOALS))}}]} result = generate_verified_rag(question, payload, complete) assert result['accepted_for_delivery'] and len(result['attempts']) == len(calls) == 1 assert short in result['answer'] and '3.' not in result['answer'] assert result['attempts'][0]['parsed_answer']['evidence_quotes'][0]['quote'] == GOALS assert result['complete'] is False and result['truncation_reasons'] == payload['truncation_reasons'] assert result['answer_coverage']['status'] == 'supported' and result['answer_coverage']['warnings'] == [] def test_short_comparison_can_cover_three_people_instead_of_dropping_one(payload): sources = [] quotes = [] for index, (name, slug) in enumerate([('Eva Nová', 'eva_nova'), ('Boris Nový', 'boris_novy'), ('Cyril Malý', 'cyril_maly')], 1): source = {**payload['sources'][0], 'source_id': f'S{index}', 'title': name, 'source_url': f'https://example.test/{slug}', 'document_path': f'pages/students/2020/{slug}/README.md'} sources.append(source) quotes.append({'source_id': source['source_id'], 'quote': '2. Implementujte vyhľadávač.'}) payload['sources'] = sources; payload['context'] = build_context_text(sources) c = rag_evidence_context('Porovnaj obsah diplomových prác osôb Eva Nová, Boris Nový a Cyril Malý.', payload) assert len(c['requirements']['people']) == 3 a = {'status': 'ok', 'claims': [], 'evidence_quotes': quotes, 'source_urls': [s['source_url'] for s in sources]} assert rag_answer_issues(a, c) == [] a['evidence_quotes'].pop() assert 'missing_bound_content_evidence' in rag_answer_issues(a, c) def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidence(payload): from tools.answer_evaluation.verified_rag import overview_excerpts extra = 'Stav:\n\n- Implementované indexovanie dokumentov.' payload['sources'][0]['text'] += '\n' + extra payload['context'] = build_context_text(payload['sources']) c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload) a = quote(GOALS) a['evidence_quotes'].append({'source_id': 'S1', 'quote': extra}) before = copy.deepcopy(a) assert rag_answer_issues(a, c) == [] units = overview_excerpts(a, c) assert len(units) == 2 and sum(map(len, units)) <= 600 assert all(any(unit in evidence['quote'] for evidence in a['evidence_quotes']) for unit in units) assert a == before def test_long_prose_excerpt_is_selected_without_inventing_a_summary(payload): from tools.answer_evaluation.verified_rag import overview_excerpts prose = 'Práca skúma vyhľadávanie v slovenských dokumentoch. ' * 18 payload['sources'][0]['text'] += '\n' + prose payload['context'] = build_context_text(payload['sources']) c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload) a = quote(prose) assert rag_answer_issues(a, c) == [] units = overview_excerpts(a, c) assert units and sum(map(len, units)) <= 600 and all(unit in prose for unit in units)