"""Technical retrieval limits must not become false claims about answer quality.""" import copy import json import pytest from tools.answers.answer_coverage import answer_coverage from tools.answers.rag_evidence import rag_evidence_context from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues, render_rag_answer from tools.rag.rag_utils import build_context_text URL = "https://example.test/eva" GOALS = "1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.\n3. Vyhodnoťte presnosť.\n4. Navrhnite zlepšenia." @pytest.fixture def payload(): sources = [ { "source_id": "S1", "title": "Eva Nová", "document_path": "pages/students/2020/eva_nova/README.md", "source_url": URL, "text": "Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\nZadanie:\n" + GOALS, } ] return { "status": "ok", "reason": None, "complete": False, "truncated": True, "truncation_reasons": ["candidate_limit", "per_document_limit", "result_limit"], "index_version": {"version": "fixture"}, "sources": sources, "context": build_context_text(sources), } def answer(): return { "status": "ok", "answer": "", "claims": [ { "person": "Eva Nová", "work_type": "diploma_thesis", "title": "Vyhľadávanie informácií", "year": 2026, } ], "evidence_quotes": [], "source_urls": [URL], } def quote(text): return { "status": "ok", "answer": "", "claims": [], "source_urls": [URL], "evidence_quotes": [{"source_id": "S1", "quote": text}], } @pytest.mark.parametrize( "complete,truncated,reasons", [ (False, True, ["candidate_limit", "per_document_limit", "result_limit"]), (None, False, []), (False, True, ["llm_context_budget"]), ], ) def test_supported_fact_is_not_an_incomplete_answer(payload, complete, truncated, reasons): payload.update(complete=complete, truncated=truncated, truncation_reasons=reasons) c = rag_evidence_context("Aký je názov a rok diplomovej práce Evy Novej?", payload) before = copy.deepcopy(c) coverage = answer_coverage(answer(), c) assert coverage["status"] == "supported" and coverage["warnings"] == [] text = render_rag_answer(answer(), c) assert "2026" in text and URL in text and "neúpln" not in text assert c == before # Never fake complete=true or remove diagnostic limits. @pytest.mark.parametrize( "question", [ "Vypíš všetky práce Evy Novej.", "Aké práce má Eva Nová?", "Uveď kompletný zoznam prác Evy Novej.", "Vypíš všetky bakalárske a diplomové práce Evy Novej.", ], ) def test_limited_record_list_still_warns(payload, question): c = rag_evidence_context(question, payload) coverage = answer_coverage(answer(), c) assert coverage["scope"] == "list" and coverage["status"] == "partial" assert "Uvedený zoznam nemusí obsahovať všetky záznamy." in render_rag_answer(answer(), c) def test_explicitly_complete_retrieval_does_not_add_a_list_warning(payload): payload.update(complete=True, truncated=False, truncation_reasons=[]) c = rag_evidence_context("Vypíš všetky práce Evy Novej.", payload) assert answer_coverage(answer(), c)["warnings"] == [] def test_quoted_title_is_not_an_exhaustive_list_request(payload): c = rag_evidence_context("Aký je rok diplomovej práce „Všetky práce v katalógu“?", payload) assert answer_coverage(answer(), c)["scope"] == "fact" def test_unknown_list_coverage_is_not_silently_exhaustive(payload): payload.update(complete=None, truncated=False, truncation_reasons=[]) c = rag_evidence_context("Aké práce má Eva Nová?", payload) assert answer_coverage(answer(), c)["warnings"] == ["list_may_be_incomplete"] def test_legacy_context_without_limit_metadata_remains_usable(payload): payload.pop("truncation_reasons") payload.pop("complete") payload.pop("truncated") c = rag_evidence_context("Aký je názov diplomovej práce Evy Novej?", payload) assert answer_coverage(answer(), c)["warnings"] == [] assert "Vyhľadávanie informácií" in render_rag_answer(answer(), c) def test_regular_assignment_can_show_all_supplied_goals_without_global_warning(payload): c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej?", payload) a = quote(GOALS) assert rag_answer_issues(a, c) == [] text = render_rag_answer(a, c) assert "1. Preskúmanie metód vyhľadávania." in text assert "4. Návrh zlepšení." in text assert "neúpln" not in text and "Zo zdroja" not in text assert answer_coverage(a, c)["scope"] == "assignment" def test_clipped_assignment_gets_specific_warning(payload): payload["sources"][0]["text"] += "\n[OBSAH ZDROJA OREZANÝ]" payload["context"] = build_context_text(payload["sources"]) c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej?", payload) a = quote("1. Preskúmajte metódy vyhľadávania.") coverage = answer_coverage(a, c) assert coverage["status"] == "partial" and coverage["warnings"] == ["assignment_text_clipped"] assert "uvedené body nemusia tvoriť celé zadanie" in render_rag_answer(a, c) def test_request_for_all_assignment_items_preserves_uncertainty(payload): c = rag_evidence_context("Uveď všetky body zadania diplomovej práce Evy Novej.", payload) assert answer_coverage(quote(GOALS), c)["warnings"] == ["assignment_completeness_unknown"] @pytest.mark.parametrize("budget", [False, True]) def test_no_answer_is_bounded_and_only_reports_actual_context_cut(payload, budget): payload["truncation_reasons"] = ["llm_context_budget"] if budget else ["result_limit"] c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej v roku 1999?", payload) a = {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []} text = render_rag_answer(a, c) assert ( "Eva Nová" in text and "v dostupných podkladoch sa nenašiel záznam diplomovej práce z roku **1999**" in text ) assert "Výpis je neúplný" not in text assert ("Časť podkladov bola pri príprave kontextu skrátená." in text) == budget assert answer_coverage(a, c)["status"] == "not_found" def test_irrelevant_dropped_source_does_not_make_cited_assignment_partial(payload): payload["sources"].append( { "source_id": "S2", "title": "Iný dokument", "source_url": "https://example.test/other", "document_path": "pages/topics/other/README.md", "text": "Iný obsah\n[OBSAH ZDROJA OREZANÝ]", } ) payload["context"] = build_context_text(payload["sources"]) payload["truncation_reasons"].append("llm_context_budget") c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej?", payload) assert answer_coverage(quote(GOALS), c)["warnings"] == [] def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload): c = rag_evidence_context("Koľko prác je evidovaných?", payload) c["aggregation"] = {"count": 2, "complete": True} a = {"status": "ok", "claims": [], "evidence_quotes": [], "source_urls": [], "count": 2} assert answer_coverage(a, c)["status"] == "supported" c["aggregation"]["complete"] = False assert "count_not_established" in answer_coverage(a, c)["warnings"] def test_overview_preserves_all_main_assignment_items_without_another_model_request(payload): question = "Čomu sa venuje diplomová práca Evy Novej?" short = "1. Preskúmanie metód vyhľadávania.\n2. Implementácia vyhľadávača." c = rag_evidence_context(question, payload) assert rag_answer_issues(quote(GOALS), c) == [] calls = [] def complete(messages): calls.append(messages) return {"choices": [{"message": {"content": json.dumps(quote(GOALS))}}]} result = generate_verified_rag(question, payload, complete) assert result["accepted_for_delivery"] and len(result["attempts"]) == len(calls) == 1 assert short in result["answer"] and "3. Vyhodnotenie presnosti." in result["answer"] assert "4. Návrh zlepšení." in result["answer"] assert result["attempts"][0]["parsed_answer"]["evidence_quotes"][0]["quote"] == GOALS assert ( result["complete"] is False and result["truncation_reasons"] == payload["truncation_reasons"] ) assert ( result["answer_coverage"]["status"] == "supported" and result["answer_coverage"]["warnings"] == [] ) def test_short_comparison_can_cover_three_people_instead_of_dropping_one(payload): sources = [] quotes = [] for index, (name, slug) in enumerate( [("Eva Nová", "eva_nova"), ("Boris Nový", "boris_novy"), ("Cyril Malý", "cyril_maly")], 1 ): source = { **payload["sources"][0], "source_id": f"S{index}", "title": name, "source_url": f"https://example.test/{slug}", "document_path": f"pages/students/2020/{slug}/README.md", } sources.append(source) quotes.append({"source_id": source["source_id"], "quote": GOALS}) payload["sources"] = sources payload["context"] = build_context_text(sources) c = rag_evidence_context( "Porovnaj obsah diplomových prác osôb Eva Nová, Boris Nový a Cyril Malý.", payload ) assert len(c["requirements"]["people"]) == 3 a = { "status": "ok", "claims": [], "evidence_quotes": quotes, "source_urls": [s["source_url"] for s in sources], } assert rag_answer_issues(a, c) == [] a["evidence_quotes"].pop() assert "missing_bound_content_evidence" in rag_answer_issues(a, c) def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidence(payload): from tools.answers.verified_rag import overview_excerpts extra = "Stav:\n\n- Implementované indexovanie dokumentov." payload["sources"][0]["text"] += "\n" + extra payload["context"] = build_context_text(payload["sources"]) c = rag_evidence_context("Čomu sa venuje diplomová práca Evy Novej?", payload) a = quote(GOALS) a["evidence_quotes"].append({"source_id": "S1", "quote": extra}) before = copy.deepcopy(a) assert rag_answer_issues(a, c) == [] units = overview_excerpts(a, c) assert len(units) == 4 and all(line in units for line in GOALS.splitlines()) assert all( any(unit in evidence["quote"] for evidence in a["evidence_quotes"]) for unit in units ) assert a == before def test_long_prose_excerpt_is_selected_without_inventing_a_summary(payload): from tools.answers.verified_rag import overview_excerpts prose = "Práca skúma vyhľadávanie v slovenských dokumentoch. " * 18 payload["sources"][0]["text"] = ( "Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\n" + prose ) payload["context"] = build_context_text(payload["sources"]) c = rag_evidence_context("Čomu sa venuje diplomová práca Evy Novej?", payload) a = quote(prose) assert rag_answer_issues(a, c) == [] units = overview_excerpts(a, c) assert units and sum(map(len, units)) <= 600 and all(unit in prose for unit in units)