"""User-facing failure distinctions and real graph projection through delivery.""" import json from contextlib import nullcontext import pytest from fastapi.testclient import TestClient import app.main as main import app.routes as routes from packages.claude import client from tools.answers import answer_delivery as delivery from tools.answers.answer_errors import MESSAGES from tools.graphrag import graph_search as gs from tools.graphrag.tests.test_graph_search import FakeSession from test_claude_packages import backend, KEY QUESTION = "Aké práce má Eva Nová?" URL = "https://example.test/eva" def graph_session(size=4): rows = [ { "person": "Eva Nová", "person_element_id": "eva", "document_path": "eva.md", "source_url": URL, "work_id": f"work:{i}", "work_element_id": f"w{i}", "work_type": "diploma_project", "year": 2024 if i < 2 else 2025, "heading": f"Diplomový projekt {i+1} " + ("2024" if i < 2 else "2025"), "title": f"Téma projektu {i+1}", } for i in range(size) ] catalog = [ {"kind": "Person", "element_id": "eva", "name": "Eva Nová", "source_document": "eva.md"} ] catalog += [ { "kind": "Work", "element_id": f"w{i}", "id": f"work:{i}", "heading": r["heading"], "title": r["title"], } for i, r in enumerate(rows) ] return FakeSession({gs.CATALOG_QUERY: catalog, gs.WORK_FACTS_QUERY: rows}) def select_graph(messages): context = json.loads(messages[1]["content"])["graph_context"] fields = ["person", "record_id", "work_type", "work_label", "title", "year"] answer = { "status": context["status"], "answer": "", "claims": [{k: r[k] for k in fields if r.get(k) is not None} for r in context["facts"]], "source_urls": [s["source_url"] for s in context["sources"] if s.get("source_url")], } return {"choices": [{"message": {"content": json.dumps(answer, ensure_ascii=False)}}]} @pytest.mark.parametrize("limit", [1, 2, 4, 5, 20]) @pytest.mark.parametrize("size", [0, 4]) def test_graph_limit_reaches_search_and_keeps_distinct_bound_records(monkeypatch, limit, size): session = graph_session(size) calls = [] def search(question, **kwargs): calls.append((question, kwargs)) return gs.search_graph_with_session(session, question, **kwargs) monkeypatch.setattr(delivery, "search_graph", search) r = delivery.deliver_answer(QUESTION, engine="graphrag", limit=limit, complete=select_graph) assert calls == [(QUESTION, {"limit": limit})] assert r["accepted_for_delivery"] and len(r["claims"]) == min(size, limit) assert len({c["record_id"] for c in r["claims"]}) == min(size, limit) for claim in r["claims"]: i = int(claim["record_id"].split(":")[-1]) assert claim["person"] == "Eva Nová" and claim["title"] == f"Téma projektu {i+1}" assert claim["year"] == (2024 if i < 2 else 2025) if size > limit: assert r["complete"] is False and r["truncated"] is True assert "fact_limit" in r["truncation_reasons"] assert "Výpis je neúplný." in r["answer"] and "Výpis môžeš zúžiť" in r["answer"] else: assert r["complete"] is True and not r["truncated"] assert "Výpis je neúplný." not in r["answer"] def test_api_passes_graph_limit_and_hides_diagnostics(monkeypatch): session = graph_session() calls = [] def search(question, **kwargs): calls.append(kwargs["limit"]) return gs.search_graph_with_session(session, question, **kwargs) monkeypatch.setattr(delivery, "search_graph", search) monkeypatch.setattr(delivery, "complete_with_configured_model", select_graph) monkeypatch.setattr(main, "embed_query", lambda _: None) with TestClient(main.app) as api: response = api.post( "/answer", json={"query": QUESTION, "engine": "graphrag", "limit": 1}, headers={"X-API-Key": KEY}, ) assert response.status_code == 200 r = response.json() assert calls == [1] and len(r["claims"]) == 1 assert r["truncated"] and "Výpis môžeš zúžiť" in r["answer"] assert "attempts" not in r and "retrieval_context" not in r assert ( api.post( "/answer", json={"query": QUESTION, "engine": "graphrag", "limit": 0}, headers={"X-API-Key": KEY}, ).status_code == 422 ) @pytest.mark.parametrize("engine", ["rag", "graphrag"]) @pytest.mark.parametrize("returned_error", [False, True]) def test_failed_retrieval_is_not_no_answer_and_never_calls_model( monkeypatch, engine, returned_error ): def search(*args, **kwargs): if returned_error: return {"status": "error", "reason": "PRIVATE_RETRIEVAL_FAILURE"} raise RuntimeError("PRIVATE_RETRIEVAL_FAILURE") monkeypatch.setattr(delivery, "search_graph", search) monkeypatch.setattr(delivery, "build_rag_context", search) monkeypatch.setattr(delivery, "index_snapshot", lambda *a, **kw: nullcontext()) r = delivery.deliver_answer( QUESTION, engine=engine, complete=lambda _: pytest.fail("No model after retrieval failure") ) assert r["status"] == "error" and r["reason"] == "retrieval_unavailable" assert not r["accepted_for_delivery"] and r["answer"] == MESSAGES["retrieval_unavailable"] assert "PRIVATE" not in json.dumps(r) @pytest.mark.parametrize("engine", ["rag", "graphrag"]) @pytest.mark.parametrize("mode", ["transport", "invalid_format", "unsupported_claim"]) def test_model_failure_is_distinct_from_missing_data(monkeypatch, mode, engine): if engine == "graphrag": monkeypatch.setattr( delivery, "search_graph", lambda q, **kw: gs.search_graph_with_session(graph_session(), q, **kw), ) else: from tools.rag.rag_utils import build_context_text source = { "title": "Eva Nová", "source_url": URL, "document_path": "pages/students/2022/eva_nova/README.md", "source_id": "S1", "text": "Dokument: Eva Nová\nSekcia: Diplomová práca 2024\n\nNázov: Ukážkový systém.", } payload = { "status": "ok", "complete": True, "truncated": False, "truncation_reasons": [], "index_version": None, "sources": [source], "context": build_context_text([source]), } monkeypatch.setattr(delivery, "build_rag_context", lambda *a, **kw: payload) monkeypatch.setattr(delivery, "index_snapshot", lambda *a, **kw: nullcontext()) monkeypatch.setattr(delivery, "expand_comparison_sources", lambda db, q, p: p) monkeypatch.setattr(delivery, "prepare_named_content", lambda db, q, p: p) calls = [] def fail(messages): calls.append(messages) if mode == "transport": raise RuntimeError("PRIVATE_MODEL_FAILURE") if mode == "invalid_format": text = "PRIVATE NOT JSON" else: text = json.dumps( { "status": "ok", "claims": [{"person": "PRIVATE WRONG PERSON", "year": 2099}], "source_urls": [URL], "answer": "", } ) return {"choices": [{"message": {"content": text}}]} r = delivery.deliver_answer(QUESTION, engine=engine, complete=fail) reason = "model_unavailable" if mode == "transport" else "answer_rejected" assert r["reason"] == reason and r["status"] == "error" and not r["accepted_for_delivery"] assert r["answer"] == MESSAGES[reason] and "spresni" not in r["answer"].casefold() assert "PRIVATE" not in r["answer"] and not r["claims"] and not r["source_urls"] assert len(calls) == (1 if mode == "transport" else 2) @pytest.mark.parametrize("engine", ["rag", "graphrag"]) def test_busy_agent_is_not_an_ambiguous_question(monkeypatch, engine): import threading monkeypatch.setattr(delivery, "_capacity", threading.BoundedSemaphore(0)) r = delivery.deliver_answer( QUESTION, engine=engine, complete=lambda _: pytest.fail("Agent is busy") ) assert r["status"] == "error" and r["reason"] == "answer_capacity_exceeded" assert r["answer"] == MESSAGES["answer_capacity_exceeded"] and not r["accepted_for_delivery"] @pytest.mark.parametrize("reason", list(MESSAGES)) def test_client_and_api_have_consistent_safe_failure_wording(backend, reason): backend["checked"].update(status="error", reason=reason, answer="PRIVATE REJECTED PROSE") r = client.query_backend(QUESTION, "rag") assert r["status"] == "error" and r["reason"] == reason assert r["answer"] == MESSAGES[reason] and not r["accepted_for_delivery"] assert "PRIVATE" not in json.dumps(r) @pytest.mark.parametrize( "status,reason,text", [ ("ambiguous", "ambiguous_work", "Ktorý projekt myslíš? Vyber projekt 1 alebo projekt 2."), ( "no_answer", "requested_aspects_not_found", "V podkladoch chýbajú údaje o riešených problémoch.", ), ("ok", None, "Doložený výsledok. V podkladoch chýbajú údaje o experimentoch."), ], ) def test_clarification_missing_evidence_and_partial_answer_stay_distinct( backend, status, reason, text ): backend["checked"].update(status=status, reason=reason, answer=text) r = client.answer_for_presentation(client.query_backend(QUESTION, "rag")) assert r["status"] == status and r["reason"] == reason and r["accepted_for_delivery"] assert r["answer"] == text