254 lines
10 KiB
Python
254 lines
10 KiB
Python
"""User-facing failure distinctions and real graph projection through delivery."""
|
|
|
|
import json
|
|
from contextlib import nullcontext
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
import app.main as main
|
|
import app.routes as routes
|
|
from packages.claude import client
|
|
from tools.answers import answer_delivery as delivery
|
|
from tools.answers.answer_errors import MESSAGES
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
from test_claude_packages import backend, KEY
|
|
|
|
QUESTION = "Aké práce má Eva Nová?"
|
|
URL = "https://example.test/eva"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolated_planning_catalog(monkeypatch):
|
|
from tools.answers import question_plan
|
|
monkeypatch.setattr(question_plan,'planning_catalog',lambda db:[{'kind':'Person','name':'Eva Nová','source_document':'eva.md'}])
|
|
monkeypatch.setattr(delivery,'index_snapshot',lambda *a,**kw:nullcontext())
|
|
|
|
|
|
def graph_session(size=4):
|
|
rows = [
|
|
{
|
|
"person": "Eva Nová",
|
|
"person_element_id": "eva",
|
|
"document_path": "eva.md",
|
|
"source_url": URL,
|
|
"work_id": f"work:{i}",
|
|
"work_element_id": f"w{i}",
|
|
"work_type": "diploma_project",
|
|
"year": 2024 if i < 2 else 2025,
|
|
"heading": f"Diplomový projekt {i+1} " + ("2024" if i < 2 else "2025"),
|
|
"title": f"Téma projektu {i+1}",
|
|
}
|
|
for i in range(size)
|
|
]
|
|
catalog = [
|
|
{"kind": "Person", "element_id": "eva", "name": "Eva Nová", "source_document": "eva.md"}
|
|
]
|
|
catalog += [
|
|
{
|
|
"kind": "Work",
|
|
"element_id": f"w{i}",
|
|
"id": f"work:{i}",
|
|
"heading": r["heading"],
|
|
"title": r["title"],
|
|
}
|
|
for i, r in enumerate(rows)
|
|
]
|
|
return FakeSession({gs.CATALOG_QUERY: catalog, gs.WORK_FACTS_QUERY: rows})
|
|
|
|
|
|
def select_graph(messages):
|
|
context = json.loads(messages[1]["content"])["graph_context"]
|
|
fields = ["person", "record_id", "work_type", "work_label", "title", "year"]
|
|
answer = {
|
|
"status": context["status"],
|
|
"answer": "",
|
|
"claims": [{k: r[k] for k in fields if r.get(k) is not None} for r in context["facts"]],
|
|
"source_urls": [s["source_url"] for s in context["sources"] if s.get("source_url")],
|
|
}
|
|
return {"choices": [{"message": {"content": json.dumps(answer, ensure_ascii=False)}}]}
|
|
|
|
|
|
@pytest.mark.parametrize("limit", [1, 2, 4, 5, 20])
|
|
@pytest.mark.parametrize("size", [0, 4])
|
|
def test_graph_limit_reaches_search_and_keeps_distinct_bound_records(monkeypatch, limit, size):
|
|
session = graph_session(size)
|
|
calls = []
|
|
|
|
def search(question, **kwargs):
|
|
calls.append((question, kwargs))
|
|
return gs.search_graph_with_session(session, question, **kwargs)
|
|
|
|
monkeypatch.setattr(delivery, "search_graph", search)
|
|
r = delivery.deliver_answer(QUESTION, engine="graphrag", limit=limit, complete=select_graph)
|
|
assert len(calls)==1 and calls[0][0]==QUESTION and calls[0][1]["limit"]==limit
|
|
assert calls[0][1]["question_plan"].intent=="metadata"
|
|
assert r["accepted_for_delivery"] and len(r["claims"]) == min(size, limit)
|
|
assert len({c["record_id"] for c in r["claims"]}) == min(size, limit)
|
|
for claim in r["claims"]:
|
|
i = int(claim["record_id"].split(":")[-1])
|
|
assert claim["person"] == "Eva Nová" and claim["title"] == f"Téma projektu {i+1}"
|
|
assert claim["year"] == (2024 if i < 2 else 2025)
|
|
if size > limit:
|
|
assert r["complete"] is False and r["truncated"] is True
|
|
assert "fact_limit" in r["truncation_reasons"]
|
|
assert "Výpis je neúplný." in r["answer"] and "Výpis môžeš zúžiť" in r["answer"]
|
|
else:
|
|
assert r["complete"] is True and not r["truncated"]
|
|
assert "Výpis je neúplný." not in r["answer"]
|
|
|
|
|
|
def test_api_passes_graph_limit_and_hides_diagnostics(monkeypatch):
|
|
session = graph_session()
|
|
calls = []
|
|
|
|
def search(question, **kwargs):
|
|
calls.append(kwargs["limit"])
|
|
return gs.search_graph_with_session(session, question, **kwargs)
|
|
|
|
monkeypatch.setattr(delivery, "search_graph", search)
|
|
monkeypatch.setattr(delivery, "complete_with_configured_model", select_graph)
|
|
monkeypatch.setattr(main, "embed_query", lambda _: None)
|
|
with TestClient(main.app) as api:
|
|
response = api.post(
|
|
"/answer",
|
|
json={"query": QUESTION, "engine": "graphrag", "limit": 1},
|
|
headers={"X-API-Key": KEY},
|
|
)
|
|
assert response.status_code == 200
|
|
r = response.json()
|
|
assert calls == [1] and len(r["claims"]) == 1
|
|
assert r["truncated"] and "Výpis môžeš zúžiť" in r["answer"]
|
|
assert "attempts" not in r and "retrieval_context" not in r
|
|
assert (
|
|
api.post(
|
|
"/answer",
|
|
json={"query": QUESTION, "engine": "graphrag", "limit": 0},
|
|
headers={"X-API-Key": KEY},
|
|
).status_code
|
|
== 422
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("engine", ["rag", "graphrag"])
|
|
@pytest.mark.parametrize("returned_error", [False, True])
|
|
def test_failed_retrieval_is_not_no_answer_and_never_calls_model(
|
|
monkeypatch, engine, returned_error
|
|
):
|
|
def search(*args, **kwargs):
|
|
if returned_error:
|
|
return {"status": "error", "reason": "PRIVATE_RETRIEVAL_FAILURE"}
|
|
raise RuntimeError("PRIVATE_RETRIEVAL_FAILURE")
|
|
|
|
monkeypatch.setattr(delivery, "search_graph", search)
|
|
monkeypatch.setattr(delivery, "build_rag_context", search)
|
|
monkeypatch.setattr(delivery, "index_snapshot", lambda *a, **kw: nullcontext())
|
|
r = delivery.deliver_answer(
|
|
QUESTION, engine=engine, complete=lambda _: pytest.fail("No model after retrieval failure")
|
|
)
|
|
assert r["status"] == "error" and r["reason"] == "retrieval_unavailable"
|
|
assert not r["accepted_for_delivery"] and r["answer"] == MESSAGES["retrieval_unavailable"]
|
|
assert "PRIVATE" not in json.dumps(r)
|
|
|
|
|
|
@pytest.mark.parametrize("engine", ["rag", "graphrag"])
|
|
@pytest.mark.parametrize("mode", ["transport", "invalid_format", "unsupported_claim"])
|
|
def test_model_failure_is_distinct_from_missing_data(monkeypatch, mode, engine):
|
|
if engine == "graphrag":
|
|
monkeypatch.setattr(
|
|
delivery,
|
|
"search_graph",
|
|
lambda q, **kw: gs.search_graph_with_session(graph_session(), q, **kw),
|
|
)
|
|
else:
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
source = {
|
|
"title": "Eva Nová",
|
|
"source_url": URL,
|
|
"document_path": "pages/students/2022/eva_nova/README.md",
|
|
"source_id": "S1",
|
|
"text": "Dokument: Eva Nová\nSekcia: Diplomová práca 2024\n\nNázov: Ukážkový systém.",
|
|
}
|
|
payload = {
|
|
"status": "ok",
|
|
"complete": True,
|
|
"truncated": False,
|
|
"truncation_reasons": [],
|
|
"index_version": None,
|
|
"sources": [source],
|
|
"context": build_context_text([source]),
|
|
}
|
|
monkeypatch.setattr(delivery, "build_rag_context", lambda *a, **kw: payload)
|
|
monkeypatch.setattr(delivery, "index_snapshot", lambda *a, **kw: nullcontext())
|
|
monkeypatch.setattr(delivery, "expand_comparison_sources", lambda db, q, p: p)
|
|
monkeypatch.setattr(delivery, "prepare_named_content", lambda db, q, p: p)
|
|
calls = []
|
|
|
|
def fail(messages):
|
|
calls.append(messages)
|
|
if mode == "transport":
|
|
raise RuntimeError("PRIVATE_MODEL_FAILURE")
|
|
if mode == "invalid_format":
|
|
text = "PRIVATE NOT JSON"
|
|
else:
|
|
text = json.dumps(
|
|
{
|
|
"status": "ok",
|
|
"claims": [{"person": "PRIVATE WRONG PERSON", "year": 2099}],
|
|
"source_urls": [URL],
|
|
"answer": "",
|
|
}
|
|
)
|
|
return {"choices": [{"message": {"content": text}}]}
|
|
|
|
r = delivery.deliver_answer(QUESTION, engine=engine, complete=fail)
|
|
reason = "model_unavailable" if mode == "transport" else "answer_rejected"
|
|
assert r["reason"] == reason and r["status"] == "error" and not r["accepted_for_delivery"]
|
|
assert r["answer"] == MESSAGES[reason] and "spresni" not in r["answer"].casefold()
|
|
assert "PRIVATE" not in r["answer"] and not r["claims"] and not r["source_urls"]
|
|
assert len(calls) == (1 if mode == "transport" else 2)
|
|
|
|
|
|
@pytest.mark.parametrize("engine", ["rag", "graphrag"])
|
|
def test_busy_agent_is_not_an_ambiguous_question(monkeypatch, engine):
|
|
import threading
|
|
|
|
monkeypatch.setattr(delivery, "_capacity", threading.BoundedSemaphore(0))
|
|
r = delivery.deliver_answer(
|
|
QUESTION, engine=engine, complete=lambda _: pytest.fail("Agent is busy")
|
|
)
|
|
assert r["status"] == "error" and r["reason"] == "answer_capacity_exceeded"
|
|
assert r["answer"] == MESSAGES["answer_capacity_exceeded"] and not r["accepted_for_delivery"]
|
|
|
|
|
|
@pytest.mark.parametrize("reason", list(MESSAGES))
|
|
def test_client_and_api_have_consistent_safe_failure_wording(backend, reason):
|
|
backend["checked"].update(status="error", reason=reason, answer="PRIVATE REJECTED PROSE")
|
|
r = client.query_backend(QUESTION, "rag")
|
|
assert r["status"] == "error" and r["reason"] == reason
|
|
assert r["answer"] == MESSAGES[reason] and not r["accepted_for_delivery"]
|
|
assert "PRIVATE" not in json.dumps(r)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"status,reason,text",
|
|
[
|
|
("ambiguous", "ambiguous_work", "Ktorý projekt myslíš? Vyber projekt 1 alebo projekt 2."),
|
|
(
|
|
"no_answer",
|
|
"requested_aspects_not_found",
|
|
"V podkladoch chýbajú údaje o riešených problémoch.",
|
|
),
|
|
("ok", None, "Doložený výsledok. V podkladoch chýbajú údaje o experimentoch."),
|
|
],
|
|
)
|
|
def test_clarification_missing_evidence_and_partial_answer_stay_distinct(
|
|
backend, status, reason, text
|
|
):
|
|
backend["checked"].update(status=status, reason=reason, answer=text)
|
|
r = client.answer_for_presentation(client.query_backend(QUESTION, "rag"))
|
|
assert r["status"] == status and r["reason"] == reason and r["accepted_for_delivery"]
|
|
assert r["answer"] == text
|