"""Behavioral checks for scope, relevant evidence, partial answers and failures.""" import copy import json import sqlite3 import pytest from tools.answers.content_selection import candidates, facets_for from tools.answers.rag_evidence import rag_evidence_context from tools.answers.rag_preparation import prepare_named_content from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues from tools.answers.answer_errors import public_failure from tools.graphrag import graph_search as gs from tools.graphrag.tests.test_graph_search import FakeSession from tools.rag.llm_context import compact_rag_payload from tools.rag.rag_utils import build_context_text from tools.retrieval.question_constraints import unsupported_constraint, requested_years @pytest.fixture def corpus(tmp_path): db = tmp_path / "index.sqlite" sources = [] with sqlite3.connect(db) as conn: conn.execute("CREATE TABLE documents(path,title,author,published)") conn.execute( "CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)" ) for i, (person, slug, kind, year, text) in enumerate( [ ( "Eva Nová", "eva_nova", "Diplomová práca", 2024, "Zadanie:\n1. Implementujte systém vyhľadávania.\nStav:\n- Konzultácia štruktúry práce.\n" "- Natrénovaný model BERT a vykonané vyhodnotenie presnosti.\n" "- Vznikol problém s tokenizérom.\nÚlohy:\n- Pripravte experiment s rôznymi tokenizérmi.", ), ( "Boris Nový", "boris_novy", "Bakalárska práca", 2023, "Zadanie:\n- Implementujte parser.\nStav:\n- Implementovaný parser pomocou Python.\n" "- Vznikla chyba pri načítaní PDF.\nÚlohy:\n- Pripravte experiment s presnosťou extrakcie.", ), ], 1, ): path = f"pages/students/2020/{slug}/README.md" conn.execute( "INSERT INTO documents VALUES(?,?,?,?)", (path, person, "Daniel Hladek", 1) ) conn.execute( "INSERT INTO chunks VALUES(?,?,?,?,?)", (path, text, json.dumps([[person, f"{kind} {year}"]]), 1, 0), ) sources.append( { "source_id": f"S{i}", "title": person, "document_path": path, "published": True, "source_url": f"https://example.test/{slug}", "text": f"Sekcia: {kind} {year}\nNázov: Ukážkový názov.", } ) raw = { "status": "ok", "reason": None, "sources": sources, "context": build_context_text(sources), "complete": False, "truncated": True, "truncation_reasons": ["result_limit"], "index_version": {"version": "fixed"}, } return db, raw def prepare(corpus, question): db, raw = corpus return compact_rag_payload(prepare_named_content(db, question, raw)) def model_selection(question, payload): c = rag_evidence_context(question, payload) return { "status": "ok", "answer": "", "claims": [], "evidence_quotes": [ {"source_id": x["source_id"], "quote": x["text"]} for x in c.get("content_candidates", []) ], "source_urls": [s["source_url"] for s in c["sources"]], } def complete(answer): return lambda _: {"choices": [{"message": {"content": json.dumps(answer)}}]} @pytest.mark.parametrize( "question,facets", [ ( "Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?", {"problems", "experiments"}, ), ("Aké metódy používa Eva Nová?", {"methods"}), ("Aké sú dosiahnuté výsledky Evy Novej?", {"results"}), ], ) def test_content_without_explicit_work_type(corpus, question, facets): payload = prepare(corpus, question) c = rag_evidence_context(question, payload) assert c["requirements"]["content"] and set(c["requirements"]["facets"]) == facets assert c["content_candidates"] def test_multiple_people_and_aspects_remain_bound(corpus): q = "Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?" payload = prepare(corpus, q) a = model_selection(q, payload) r = generate_verified_rag(q, payload, complete(a)) assert r["accepted_for_delivery"] and r["status"] == "ok" text = r["answer"] eva, boris = text.split("Boris Nový — bakalárska práca") assert "tokenizérom" in eva and "PDF" not in eva assert "PDF" in boris and "tokenizérmi" not in boris assert text.count("**Riešené problémy:**") == 2 assert text.count("**Plánované experimenty a vyhodnotenie:**") == 2 def test_general_problem_question_preserves_both_work_scopes(corpus): db, raw = corpus source = raw["sources"][0] with sqlite3.connect(db) as conn: conn.execute( "INSERT INTO chunks VALUES(?,?,?,?,?)", ( source["document_path"], "Stav:\nVznikol iný problém.", json.dumps([["Eva Nová", "Bakalárska práca 2022"]]), 1, 1, ), ) q = "Aké problémy mala Eva Nová počas práce?" p = prepare(corpus, q) assert p["status"] == "ok" assert {scope["year"] for scope in p["work_scopes"]} == {2022, 2024} assert "Vznikol iný problém" in p["context"] and "tokenizérom" in p["context"] def test_explicit_type_and_year_resolve_scope(corpus): db, raw = corpus with sqlite3.connect(db) as conn: conn.execute( "INSERT INTO chunks VALUES(?,?,?,?,?)", ( raw["sources"][0]["document_path"], "Stav:\nChyba v inom projekte Cassandra.", json.dumps([["Eva Nová", "Diplomový projekt 2022"]]), 1, 1, ), ) q = "Aké problémy mala Eva Nová pri diplomovej práci v roku 2024?" p = prepare(corpus, q) assert p["status"] == "ok" and "Cassandra" not in p["context"] def test_year_condition_without_the_words_v_roku_is_preserved(corpus): q = "Aké problémy mala Eva Nová pri diplomovej práci 1999?" p = prepare(corpus, q) c = rag_evidence_context(q, p) assert c["requirements"]["years"] == [1999] and not c.get("content_candidates") r = generate_verified_rag(q, p, lambda _: pytest.fail("No model for missing scoped evidence")) assert ( r["accepted_for_delivery"] and r["status"] == "no_answer" and "tokenizérom" not in r["answer"] ) assert requested_years("Práca „Model 2020“ z roku 2024") == [2024] def test_named_source_can_be_loaded_when_absent_from_top_results(corpus): db, raw = corpus raw["sources"] = raw["sources"][:1] raw["context"] = build_context_text(raw["sources"]) p = prepare(corpus, "Aké problémy riešili Eva Nová a Boris Nový?") assert {s["title"] for s in p["sources"]} == {"Eva Nová", "Boris Nový"} def test_preparation_does_not_change_the_original_retrieval(corpus): db, raw = corpus before = copy.deepcopy(raw) prepare_named_content(db, "Aké problémy riešili Eva Nová a Boris Nový?", raw) assert raw == before def test_source_budget_cannot_silently_remove_a_requested_person(corpus): db, raw = corpus q = "Aké problémy riešili Eva Nová a Boris Nový?" p = prepare_named_content(db, q, raw) p["sources"][0]["text"] += "\n" + "Veľmi dlhý zdrojový text. " * 1000 compact = compact_rag_payload(p, max_tokens=1100) c = rag_evidence_context(q, compact) assert compact["truncated"] assert set(c["requirements"]["people"]) == {"Eva Nová", "Boris Nový"} def test_candidates_are_actual_bound_literal_evidence(corpus): q = "Aké problémy riešili Eva Nová a Boris Nový?" c = rag_evidence_context(q, prepare(corpus, q)) for candidate in candidates(c): assert any( all(candidate[k] == p[k] for k in ["person", "work_type", "year", "source_id"]) and candidate["text"] in p["text"] for p in c["content_passages"] ) def test_unknown_second_subject_is_not_silently_ignored(corpus): p = prepare(corpus, "Aké problémy riešili študenti Eva Nová a Boris Nxxxxx?") assert p["status"] == "ambiguous" and p["reason"] == "unresolved_subject" @pytest.mark.parametrize( "question", [ "Aké problémy riešili Eva Nová a Xaver Neexistujúci?", "Aké problémy riešil Boris Nxxxxx?", ], ) def test_unresolved_subject_cannot_be_replaced_by_available_profiles(corpus, question): p = prepare(corpus, question) assert p["status"] == "ambiguous" and p["reason"] == "unresolved_subject" def test_content_preparation_uses_shared_typo_resolution_and_keeps_notice(corpus): p = prepare(corpus, "Aké problémy riešili študenti Eva Nová a Boris Noyy?") assert p["status"] == "ok" and p["named_subjects"] == ["Eva Nová", "Boris Nový"] assert p["resolved_query"] == "Aké problémy riešili študenti Eva Nová a Boris Nový?" assert any("Boris Nový" in n for n in p["name_resolution"]["notices"]) def test_content_preparation_does_not_treat_per_person_instructions_as_names(corpus): q = ( "Porovnaj výsledky Evy Novej a Borisa Nového. " "Pri každej osobe príslušné výsledky uveď samostatne. Pridaj zdroje." ) p = prepare(corpus, q) assert p["status"] == "ok" and p["named_subjects"] == ["Eva Nová", "Boris Nový"] assert p["name_resolution"]["status"] == "ok" def test_partial_answer_names_the_missing_aspect(corpus): db, raw = corpus with sqlite3.connect(db) as conn: conn.execute( "UPDATE chunks SET text='Stav:\nVznikol problém s parserom.' WHERE document_path LIKE '%boris_novy%'" ) q = "Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?" p = prepare(corpus, q) r = generate_verified_rag(q, p, complete(model_selection(q, p))) assert r["accepted_for_delivery"] and r["answer_coverage"]["status"] == "partial" assert "V poskytnutých podkladoch chýbajú údaje o plánovaných experimentoch." in r["answer"] assert "parserom" in r["answer"] def test_all_aspects_absent_is_no_answer_not_rejection(corpus): db, raw = corpus with sqlite3.connect(db) as conn: conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.'") q = "Aké problémy riešili Eva Nová a Boris Nový?" r = generate_verified_rag( q, prepare(corpus, q), lambda _: pytest.fail("No model for a bounded empty result") ) assert r["status"] == "no_answer" and r["accepted_for_delivery"] and not r["claims"] def test_known_year_is_returned_when_requested_title_is_missing(corpus): _, raw = corpus source = raw["sources"][0] source["text"] = "Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser." raw["sources"] = [source] raw["context"] = build_context_text(raw["sources"]) q = "Uveď názov a rok diplomovej práce Evy Novej." a = { "status": "ok", "answer": "", "claims": [{"person": "Eva Nová", "work_type": "diploma_thesis", "year": 2024}], "evidence_quotes": [], "source_urls": [source["source_url"]], } r = generate_verified_rag(q, raw, complete(a)) assert r["accepted_for_delivery"] and "2024" in r["answer"] assert "názov práce nie je v poskytnutých podkladoch doložený" in r["answer"] assert r["answer_coverage"]["status"] == "partial" and not r["claims"][0].get("title") def test_concrete_results_win_over_administrative_notes(corpus): q = "Aké výsledky dosiahla Eva Nová?" p = prepare(corpus, q) c = rag_evidence_context(q, p) text = "\n".join(x["text"] for x in c["content_candidates"]) assert "Natrénovaný model BERT" in text r = generate_verified_rag(q, p, complete(model_selection(q, p))) assert r["accepted_for_delivery"] and "Natrénovaný model BERT" in r["answer"] assert "Konzultácia" not in r["answer"] def test_duplicate_quotes_and_overlapping_chunks_do_not_duplicate_output(corpus): q = "Aké výsledky dosiahla Eva Nová?" p = prepare(corpus, q) a = model_selection(q, p) a["evidence_quotes"] *= 2 r = generate_verified_rag(q, p, complete(a)) assert r["accepted_for_delivery"] and r["answer"].count("Natrénovaný model BERT") == 1 assert "Ďalšie podklady:" not in r["answer"] and "Ďalšie doložené podklady" not in r["answer"] def test_plans_cannot_answer_actual_use_or_completed_experiments(): assert "methods" not in facets_for( "Použite metódu BERT.", "task", "Aké metódy skutočne používal?" ) assert "experiments" not in facets_for( "Pripravte experiment.", "task", "Aké experimenty vykonal?" ) assert "results" not in facets_for( "Cieľom je implementovať systém.", "progress", "Aké dosiahol výsledky?" ) def test_method_answer_does_not_add_valid_but_unrequested_profile_metadata(corpus): _, raw = corpus raw["sources"][0]["author"] = "Daniel Hladek" q = "Aké metódy používala Eva Nová pri diplomovej práci v roku 2024? Uveď aj rok práce." p = prepare(corpus, q) a = model_selection(q, p) a["claims"] = [ { "person": "Eva Nová", "work_type": "diploma_thesis", "year": 2024, "document_author": "Daniel Hladek", } ] r = generate_verified_rag(q, p, complete(a)) assert r["accepted_for_delivery"] and "BERT" in r["answer"] assert "Daniel Hladek" not in r["answer"] and "autor dokumentu" not in r["answer"] assert r["answer"].count("2024") == 1 @pytest.mark.parametrize( "question", [ "Práce od roku 2020 do roku 2024?", "Práce pred rokom 2020?", "Študenti s aspoň 3 prácami?", "Práce okrem diplomových?", "Študenti bez tagu RAG?", "V ktorom roku Ján Holp začal pracovať na diplomovej práci?", "Kedy Ján Holp obhájil diplomovú prácu?", ], ) def test_unsupported_operators_are_not_ignored(question): assert unsupported_constraint(question) result = gs.search_graph_with_session(FakeSession({}), question) assert ( result["status"] == "ambiguous" and result["reason"] == "unsupported_constraint" and not result["facts"] ) @pytest.mark.parametrize( "question", [ "Práca v roku 1999?", "Aký je názov práce „Od 2020 do 2024“?", "Aké práce alebo projekty má Eva Nová?", ], ) def test_supported_year_and_work_alternative_are_not_blocked(question): assert unsupported_constraint(question) is None @pytest.mark.parametrize( "reason", ["model_unavailable", "retrieval_unavailable", "answer_rejected", "answer_capacity_exceeded"], ) def test_public_errors_are_distinct_and_do_not_leak_rejected_text(reason): r = public_failure( { "reason": reason, "answer": "SECRET YEAR 2099", "claims": [{"person": "Eva"}], "source_urls": ["SECRET"], } ) assert r["reason"] == reason and not r["accepted_for_delivery"] assert "SECRET" not in r["answer"] and "2099" not in r["answer"] assert not r["claims"] and not r["source_urls"] def test_retrieval_failure_is_not_misreported_as_bad_model_answer(monkeypatch): import tools.answers.answer_delivery as delivery def unavailable(_): raise RuntimeError("SECRET backend address") from contextlib import nullcontext from tools.answers import question_plan monkeypatch.setattr(question_plan,'planning_catalog',lambda db:[{'kind':'Person','name':'Eva Nová','source_document':'eva.md'}]) monkeypatch.setattr(delivery,'index_snapshot',lambda *a,**kw:nullcontext()) monkeypatch.setattr(delivery, "search_graph", unavailable) r = delivery.deliver_answer( "Aké práce má Eva Nová?", engine="graphrag", complete=lambda _: pytest.fail("No model call after failed retrieval"), ) assert r["status"] == "error" and r["reason"] == "retrieval_unavailable" assert "SECRET" not in r["answer"]