285 lines
11 KiB
Python
285 lines
11 KiB
Python
"""Technical retrieval limits must not become false claims about answer quality."""
|
|
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.answers.answer_coverage import answer_coverage
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues, render_rag_answer
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
URL = "https://example.test/eva"
|
|
GOALS = "1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.\n3. Vyhodnoťte presnosť.\n4. Navrhnite zlepšenia."
|
|
|
|
|
|
@pytest.fixture
|
|
def payload():
|
|
sources = [
|
|
{
|
|
"source_id": "S1",
|
|
"title": "Eva Nová",
|
|
"document_path": "pages/students/2020/eva_nova/README.md",
|
|
"source_url": URL,
|
|
"text": "Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\nZadanie:\n"
|
|
+ GOALS,
|
|
}
|
|
]
|
|
return {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"complete": False,
|
|
"truncated": True,
|
|
"truncation_reasons": ["candidate_limit", "per_document_limit", "result_limit"],
|
|
"index_version": {"version": "fixture"},
|
|
"sources": sources,
|
|
"context": build_context_text(sources),
|
|
}
|
|
|
|
|
|
def answer():
|
|
return {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Nová",
|
|
"work_type": "diploma_thesis",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
}
|
|
],
|
|
"evidence_quotes": [],
|
|
"source_urls": [URL],
|
|
}
|
|
|
|
|
|
def quote(text):
|
|
return {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"source_urls": [URL],
|
|
"evidence_quotes": [{"source_id": "S1", "quote": text}],
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"complete,truncated,reasons",
|
|
[
|
|
(False, True, ["candidate_limit", "per_document_limit", "result_limit"]),
|
|
(None, False, []),
|
|
(False, True, ["llm_context_budget"]),
|
|
],
|
|
)
|
|
def test_supported_fact_is_not_an_incomplete_answer(payload, complete, truncated, reasons):
|
|
payload.update(complete=complete, truncated=truncated, truncation_reasons=reasons)
|
|
c = rag_evidence_context("Aký je názov a rok diplomovej práce Evy Novej?", payload)
|
|
before = copy.deepcopy(c)
|
|
coverage = answer_coverage(answer(), c)
|
|
assert coverage["status"] == "supported" and coverage["warnings"] == []
|
|
text = render_rag_answer(answer(), c)
|
|
assert "2026" in text and URL in text and "neúpln" not in text
|
|
assert c == before # Never fake complete=true or remove diagnostic limits.
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
[
|
|
"Vypíš všetky práce Evy Novej.",
|
|
"Aké práce má Eva Nová?",
|
|
"Uveď kompletný zoznam prác Evy Novej.",
|
|
"Vypíš všetky bakalárske a diplomové práce Evy Novej.",
|
|
],
|
|
)
|
|
def test_limited_record_list_still_warns(payload, question):
|
|
c = rag_evidence_context(question, payload)
|
|
coverage = answer_coverage(answer(), c)
|
|
assert coverage["scope"] == "list" and coverage["status"] == "partial"
|
|
assert "Uvedený zoznam nemusí obsahovať všetky záznamy." in render_rag_answer(answer(), c)
|
|
|
|
|
|
def test_explicitly_complete_retrieval_does_not_add_a_list_warning(payload):
|
|
payload.update(complete=True, truncated=False, truncation_reasons=[])
|
|
c = rag_evidence_context("Vypíš všetky práce Evy Novej.", payload)
|
|
assert answer_coverage(answer(), c)["warnings"] == []
|
|
|
|
|
|
def test_quoted_title_is_not_an_exhaustive_list_request(payload):
|
|
c = rag_evidence_context("Aký je rok diplomovej práce „Všetky práce v katalógu“?", payload)
|
|
assert answer_coverage(answer(), c)["scope"] == "fact"
|
|
|
|
|
|
def test_unknown_list_coverage_is_not_silently_exhaustive(payload):
|
|
payload.update(complete=None, truncated=False, truncation_reasons=[])
|
|
c = rag_evidence_context("Aké práce má Eva Nová?", payload)
|
|
assert answer_coverage(answer(), c)["warnings"] == ["list_may_be_incomplete"]
|
|
|
|
|
|
def test_legacy_context_without_limit_metadata_remains_usable(payload):
|
|
payload.pop("truncation_reasons")
|
|
payload.pop("complete")
|
|
payload.pop("truncated")
|
|
c = rag_evidence_context("Aký je názov diplomovej práce Evy Novej?", payload)
|
|
assert answer_coverage(answer(), c)["warnings"] == []
|
|
assert "Vyhľadávanie informácií" in render_rag_answer(answer(), c)
|
|
|
|
|
|
def test_regular_assignment_can_show_all_supplied_goals_without_global_warning(payload):
|
|
c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej?", payload)
|
|
a = quote(GOALS)
|
|
assert rag_answer_issues(a, c) == []
|
|
text = render_rag_answer(a, c)
|
|
assert "1. Preskúmanie metód vyhľadávania." in text
|
|
assert "4. Návrh zlepšení." in text
|
|
assert "neúpln" not in text and "Zo zdroja" not in text
|
|
assert answer_coverage(a, c)["scope"] == "assignment"
|
|
|
|
|
|
def test_clipped_assignment_gets_specific_warning(payload):
|
|
payload["sources"][0]["text"] += "\n[OBSAH ZDROJA OREZANÝ]"
|
|
payload["context"] = build_context_text(payload["sources"])
|
|
c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej?", payload)
|
|
a = quote("1. Preskúmajte metódy vyhľadávania.")
|
|
coverage = answer_coverage(a, c)
|
|
assert coverage["status"] == "partial" and coverage["warnings"] == ["assignment_text_clipped"]
|
|
assert "uvedené body nemusia tvoriť celé zadanie" in render_rag_answer(a, c)
|
|
|
|
|
|
def test_request_for_all_assignment_items_preserves_uncertainty(payload):
|
|
c = rag_evidence_context("Uveď všetky body zadania diplomovej práce Evy Novej.", payload)
|
|
assert answer_coverage(quote(GOALS), c)["warnings"] == ["assignment_completeness_unknown"]
|
|
|
|
|
|
@pytest.mark.parametrize("budget", [False, True])
|
|
def test_no_answer_is_bounded_and_only_reports_actual_context_cut(payload, budget):
|
|
payload["truncation_reasons"] = ["llm_context_budget"] if budget else ["result_limit"]
|
|
c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej v roku 1999?", payload)
|
|
a = {"status": "no_answer", "claims": [], "evidence_quotes": [], "source_urls": []}
|
|
text = render_rag_answer(a, c)
|
|
assert (
|
|
"Eva Nová" in text
|
|
and "v dostupných podkladoch sa nenašiel záznam diplomovej práce z roku **1999**" in text
|
|
)
|
|
assert "Výpis je neúplný" not in text
|
|
assert ("Časť podkladov bola pri príprave kontextu skrátená." in text) == budget
|
|
assert answer_coverage(a, c)["status"] == "not_found"
|
|
|
|
|
|
def test_irrelevant_dropped_source_does_not_make_cited_assignment_partial(payload):
|
|
payload["sources"].append(
|
|
{
|
|
"source_id": "S2",
|
|
"title": "Iný dokument",
|
|
"source_url": "https://example.test/other",
|
|
"document_path": "pages/topics/other/README.md",
|
|
"text": "Iný obsah\n[OBSAH ZDROJA OREZANÝ]",
|
|
}
|
|
)
|
|
payload["context"] = build_context_text(payload["sources"])
|
|
payload["truncation_reasons"].append("llm_context_budget")
|
|
c = rag_evidence_context("Aké je zadanie diplomovej práce Evy Novej?", payload)
|
|
assert answer_coverage(quote(GOALS), c)["warnings"] == []
|
|
|
|
|
|
def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload):
|
|
c = rag_evidence_context("Koľko prác je evidovaných?", payload)
|
|
c["aggregation"] = {"count": 2, "complete": True}
|
|
a = {"status": "ok", "claims": [], "evidence_quotes": [], "source_urls": [], "count": 2}
|
|
assert answer_coverage(a, c)["status"] == "supported"
|
|
c["aggregation"]["complete"] = False
|
|
assert "count_not_established" in answer_coverage(a, c)["warnings"]
|
|
|
|
|
|
def test_overview_preserves_all_main_assignment_items_without_another_model_request(payload):
|
|
question = "Čomu sa venuje diplomová práca Evy Novej?"
|
|
short = "1. Preskúmanie metód vyhľadávania.\n2. Implementácia vyhľadávača."
|
|
c = rag_evidence_context(question, payload)
|
|
assert rag_answer_issues(quote(GOALS), c) == []
|
|
calls = []
|
|
|
|
def complete(messages):
|
|
calls.append(messages)
|
|
return {"choices": [{"message": {"content": json.dumps(quote(GOALS))}}]}
|
|
|
|
result = generate_verified_rag(question, payload, complete)
|
|
assert result["accepted_for_delivery"] and len(result["attempts"]) == len(calls) == 1
|
|
assert short in result["answer"] and "3. Vyhodnotenie presnosti." in result["answer"]
|
|
assert "4. Návrh zlepšení." in result["answer"]
|
|
assert result["attempts"][0]["parsed_answer"]["evidence_quotes"][0]["quote"] == GOALS
|
|
assert (
|
|
result["complete"] is False
|
|
and result["truncation_reasons"] == payload["truncation_reasons"]
|
|
)
|
|
assert (
|
|
result["answer_coverage"]["status"] == "supported"
|
|
and result["answer_coverage"]["warnings"] == []
|
|
)
|
|
|
|
|
|
def test_short_comparison_can_cover_three_people_instead_of_dropping_one(payload):
|
|
sources = []
|
|
quotes = []
|
|
for index, (name, slug) in enumerate(
|
|
[("Eva Nová", "eva_nova"), ("Boris Nový", "boris_novy"), ("Cyril Malý", "cyril_maly")], 1
|
|
):
|
|
source = {
|
|
**payload["sources"][0],
|
|
"source_id": f"S{index}",
|
|
"title": name,
|
|
"source_url": f"https://example.test/{slug}",
|
|
"document_path": f"pages/students/2020/{slug}/README.md",
|
|
}
|
|
sources.append(source)
|
|
quotes.append({"source_id": source["source_id"], "quote": GOALS})
|
|
payload["sources"] = sources
|
|
payload["context"] = build_context_text(sources)
|
|
c = rag_evidence_context(
|
|
"Porovnaj obsah diplomových prác osôb Eva Nová, Boris Nový a Cyril Malý.", payload
|
|
)
|
|
assert len(c["requirements"]["people"]) == 3
|
|
a = {
|
|
"status": "ok",
|
|
"claims": [],
|
|
"evidence_quotes": quotes,
|
|
"source_urls": [s["source_url"] for s in sources],
|
|
}
|
|
assert rag_answer_issues(a, c) == []
|
|
a["evidence_quotes"].pop()
|
|
assert "missing_bound_content_evidence" in rag_answer_issues(a, c)
|
|
|
|
|
|
def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidence(payload):
|
|
from tools.answers.verified_rag import overview_excerpts
|
|
|
|
extra = "Stav:\n\n- Implementované indexovanie dokumentov."
|
|
payload["sources"][0]["text"] += "\n" + extra
|
|
payload["context"] = build_context_text(payload["sources"])
|
|
c = rag_evidence_context("Čomu sa venuje diplomová práca Evy Novej?", payload)
|
|
a = quote(GOALS)
|
|
a["evidence_quotes"].append({"source_id": "S1", "quote": extra})
|
|
before = copy.deepcopy(a)
|
|
assert rag_answer_issues(a, c) == []
|
|
units = overview_excerpts(a, c)
|
|
assert len(units) == 4 and all(line in units for line in GOALS.splitlines())
|
|
assert all(
|
|
any(unit in evidence["quote"] for evidence in a["evidence_quotes"]) for unit in units
|
|
)
|
|
assert a == before
|
|
|
|
|
|
def test_long_prose_excerpt_is_selected_without_inventing_a_summary(payload):
|
|
from tools.answers.verified_rag import overview_excerpts
|
|
|
|
prose = "Práca skúma vyhľadávanie v slovenských dokumentoch. " * 18
|
|
payload["sources"][0]["text"] = (
|
|
"Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\n" + prose
|
|
)
|
|
payload["context"] = build_context_text(payload["sources"])
|
|
c = rag_evidence_context("Čomu sa venuje diplomová práca Evy Novej?", payload)
|
|
a = quote(prose)
|
|
assert rag_answer_issues(a, c) == []
|
|
units = overview_excerpts(a, c)
|
|
assert units and sum(map(len, units)) <= 600 and all(unit in prose for unit in units)
|