426 lines
16 KiB
Python
426 lines
16 KiB
Python
"""Behavioral checks for scope, relevant evidence, partial answers and failures."""
|
|
|
|
import copy
|
|
import json
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from tools.answers.content_selection import candidates, facets_for
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.answers.rag_preparation import prepare_named_content
|
|
from tools.answers.verified_rag import generate_verified_rag, rag_answer_issues
|
|
from tools.answers.answer_errors import public_failure
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
from tools.rag.llm_context import compact_rag_payload
|
|
from tools.rag.rag_utils import build_context_text
|
|
from tools.retrieval.question_constraints import unsupported_constraint, requested_years
|
|
|
|
|
|
@pytest.fixture
|
|
def corpus(tmp_path):
|
|
db = tmp_path / "index.sqlite"
|
|
sources = []
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute("CREATE TABLE documents(path,title,author,published)")
|
|
conn.execute(
|
|
"CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)"
|
|
)
|
|
for i, (person, slug, kind, year, text) in enumerate(
|
|
[
|
|
(
|
|
"Eva Nová",
|
|
"eva_nova",
|
|
"Diplomová práca",
|
|
2024,
|
|
"Zadanie:\n1. Implementujte systém vyhľadávania.\nStav:\n- Konzultácia štruktúry práce.\n"
|
|
"- Natrénovaný model BERT a vykonané vyhodnotenie presnosti.\n"
|
|
"- Vznikol problém s tokenizérom.\nÚlohy:\n- Pripravte experiment s rôznymi tokenizérmi.",
|
|
),
|
|
(
|
|
"Boris Nový",
|
|
"boris_novy",
|
|
"Bakalárska práca",
|
|
2023,
|
|
"Zadanie:\n- Implementujte parser.\nStav:\n- Implementovaný parser pomocou Python.\n"
|
|
"- Vznikla chyba pri načítaní PDF.\nÚlohy:\n- Pripravte experiment s presnosťou extrakcie.",
|
|
),
|
|
],
|
|
1,
|
|
):
|
|
path = f"pages/students/2020/{slug}/README.md"
|
|
conn.execute(
|
|
"INSERT INTO documents VALUES(?,?,?,?)", (path, person, "Daniel Hladek", 1)
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(path, text, json.dumps([[person, f"{kind} {year}"]]), 1, 0),
|
|
)
|
|
sources.append(
|
|
{
|
|
"source_id": f"S{i}",
|
|
"title": person,
|
|
"document_path": path,
|
|
"published": True,
|
|
"source_url": f"https://example.test/{slug}",
|
|
"text": f"Sekcia: {kind} {year}\nNázov: Ukážkový názov.",
|
|
}
|
|
)
|
|
raw = {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"sources": sources,
|
|
"context": build_context_text(sources),
|
|
"complete": False,
|
|
"truncated": True,
|
|
"truncation_reasons": ["result_limit"],
|
|
"index_version": {"version": "fixed"},
|
|
}
|
|
return db, raw
|
|
|
|
|
|
def prepare(corpus, question):
|
|
db, raw = corpus
|
|
return compact_rag_payload(prepare_named_content(db, question, raw))
|
|
|
|
|
|
def model_selection(question, payload):
|
|
c = rag_evidence_context(question, payload)
|
|
return {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [],
|
|
"evidence_quotes": [
|
|
{"source_id": x["source_id"], "quote": x["text"]}
|
|
for x in c.get("content_candidates", [])
|
|
],
|
|
"source_urls": [s["source_url"] for s in c["sources"]],
|
|
}
|
|
|
|
|
|
def complete(answer):
|
|
return lambda _: {"choices": [{"message": {"content": json.dumps(answer)}}]}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question,facets",
|
|
[
|
|
(
|
|
"Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?",
|
|
{"problems", "experiments"},
|
|
),
|
|
("Aké metódy používa Eva Nová?", {"methods"}),
|
|
("Aké sú dosiahnuté výsledky Evy Novej?", {"results"}),
|
|
],
|
|
)
|
|
def test_content_without_explicit_work_type(corpus, question, facets):
|
|
payload = prepare(corpus, question)
|
|
c = rag_evidence_context(question, payload)
|
|
assert c["requirements"]["content"] and set(c["requirements"]["facets"]) == facets
|
|
assert c["content_candidates"]
|
|
|
|
|
|
def test_multiple_people_and_aspects_remain_bound(corpus):
|
|
q = "Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?"
|
|
payload = prepare(corpus, q)
|
|
a = model_selection(q, payload)
|
|
r = generate_verified_rag(q, payload, complete(a))
|
|
assert r["accepted_for_delivery"] and r["status"] == "ok"
|
|
text = r["answer"]
|
|
eva, boris = text.split("Boris Nový — bakalárska práca")
|
|
assert "tokenizérom" in eva and "PDF" not in eva
|
|
assert "PDF" in boris and "tokenizérmi" not in boris
|
|
assert text.count("**Riešené problémy:**") == 2
|
|
assert text.count("**Plánované experimenty a vyhodnotenie:**") == 2
|
|
|
|
|
|
def test_ambiguous_work_is_a_clarification_without_model(corpus):
|
|
db, raw = corpus
|
|
source = raw["sources"][0]
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(
|
|
source["document_path"],
|
|
"Stav:\nVznikol iný problém.",
|
|
json.dumps([["Eva Nová", "Bakalárska práca 2022"]]),
|
|
1,
|
|
1,
|
|
),
|
|
)
|
|
q = "Aké problémy mala Eva Nová počas práce?"
|
|
p = prepare(corpus, q)
|
|
r = generate_verified_rag(q, p, lambda _: pytest.fail("No model call for a clarification"))
|
|
assert r["status"] == "ambiguous" and r["reason"] == "ambiguous_work"
|
|
assert "2022" in r["answer"] and "2024" in r["answer"]
|
|
|
|
|
|
def test_explicit_type_and_year_resolve_scope(corpus):
|
|
db, raw = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"INSERT INTO chunks VALUES(?,?,?,?,?)",
|
|
(
|
|
raw["sources"][0]["document_path"],
|
|
"Stav:\nChyba v inom projekte Cassandra.",
|
|
json.dumps([["Eva Nová", "Diplomový projekt 2022"]]),
|
|
1,
|
|
1,
|
|
),
|
|
)
|
|
q = "Aké problémy mala Eva Nová pri diplomovej práci v roku 2024?"
|
|
p = prepare(corpus, q)
|
|
assert p["status"] == "ok" and "Cassandra" not in p["context"]
|
|
|
|
|
|
def test_year_condition_without_the_words_v_roku_is_preserved(corpus):
|
|
q = "Aké problémy mala Eva Nová pri diplomovej práci 1999?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
assert c["requirements"]["years"] == [1999] and not c.get("content_candidates")
|
|
r = generate_verified_rag(q, p, lambda _: pytest.fail("No model for missing scoped evidence"))
|
|
assert (
|
|
r["accepted_for_delivery"]
|
|
and r["status"] == "no_answer"
|
|
and "tokenizérom" not in r["answer"]
|
|
)
|
|
assert requested_years("Práca „Model 2020“ z roku 2024") == [2024]
|
|
|
|
|
|
def test_named_source_can_be_loaded_when_absent_from_top_results(corpus):
|
|
db, raw = corpus
|
|
raw["sources"] = raw["sources"][:1]
|
|
raw["context"] = build_context_text(raw["sources"])
|
|
p = prepare(corpus, "Aké problémy riešili Eva Nová a Boris Nový?")
|
|
assert {s["title"] for s in p["sources"]} == {"Eva Nová", "Boris Nový"}
|
|
|
|
|
|
def test_preparation_does_not_change_the_original_retrieval(corpus):
|
|
db, raw = corpus
|
|
before = copy.deepcopy(raw)
|
|
prepare_named_content(db, "Aké problémy riešili Eva Nová a Boris Nový?", raw)
|
|
assert raw == before
|
|
|
|
|
|
def test_source_budget_cannot_silently_remove_a_requested_person(corpus):
|
|
db, raw = corpus
|
|
q = "Aké problémy riešili Eva Nová a Boris Nový?"
|
|
p = prepare_named_content(db, q, raw)
|
|
p["sources"][0]["text"] += "\n" + "Veľmi dlhý zdrojový text. " * 1000
|
|
compact = compact_rag_payload(p, max_tokens=1100)
|
|
c = rag_evidence_context(q, compact)
|
|
assert compact["truncated"]
|
|
assert set(c["requirements"]["people"]) == {"Eva Nová", "Boris Nový"}
|
|
|
|
|
|
def test_candidates_are_actual_bound_literal_evidence(corpus):
|
|
q = "Aké problémy riešili Eva Nová a Boris Nový?"
|
|
c = rag_evidence_context(q, prepare(corpus, q))
|
|
for candidate in candidates(c):
|
|
assert any(
|
|
all(candidate[k] == p[k] for k in ["person", "work_type", "year", "source_id"])
|
|
and candidate["text"] in p["text"]
|
|
for p in c["content_passages"]
|
|
)
|
|
|
|
|
|
def test_unknown_second_subject_is_not_silently_ignored(corpus):
|
|
p = prepare(corpus, "Aké problémy riešili študenti Eva Nová a Boris Nxxxxx?")
|
|
assert p["status"] == "ambiguous" and p["reason"] == "unresolved_subject"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
[
|
|
"Aké problémy riešili Eva Nová a Xaver Neexistujúci?",
|
|
"Aké problémy riešil Boris Nxxxxx?",
|
|
],
|
|
)
|
|
def test_unresolved_subject_cannot_be_replaced_by_available_profiles(corpus, question):
|
|
p = prepare(corpus, question)
|
|
assert p["status"] == "ambiguous" and p["reason"] == "unresolved_subject"
|
|
|
|
|
|
def test_content_preparation_uses_shared_typo_resolution_and_keeps_notice(corpus):
|
|
p = prepare(corpus, "Aké problémy riešili študenti Eva Nová a Boris Noyy?")
|
|
assert p["status"] == "ok" and p["named_subjects"] == ["Eva Nová", "Boris Nový"]
|
|
assert p["resolved_query"] == "Aké problémy riešili študenti Eva Nová a Boris Nový?"
|
|
assert any("Boris Nový" in n for n in p["name_resolution"]["notices"])
|
|
|
|
|
|
def test_content_preparation_does_not_treat_per_person_instructions_as_names(corpus):
|
|
q = (
|
|
"Porovnaj výsledky Evy Novej a Borisa Nového. "
|
|
"Pri každej osobe príslušné výsledky uveď samostatne. Pridaj zdroje."
|
|
)
|
|
p = prepare(corpus, q)
|
|
assert p["status"] == "ok" and p["named_subjects"] == ["Eva Nová", "Boris Nový"]
|
|
assert p["name_resolution"]["status"] == "ok"
|
|
|
|
|
|
def test_partial_answer_names_the_missing_aspect(corpus):
|
|
db, raw = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute(
|
|
"UPDATE chunks SET text='Stav:\nVznikol problém s parserom.' WHERE document_path LIKE '%boris_novy%'"
|
|
)
|
|
q = "Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?"
|
|
p = prepare(corpus, q)
|
|
r = generate_verified_rag(q, p, complete(model_selection(q, p)))
|
|
assert r["accepted_for_delivery"] and r["answer_coverage"]["status"] == "partial"
|
|
assert "V poskytnutých podkladoch chýbajú údaje o plánovaných experimentoch." in r["answer"]
|
|
assert "parserom" in r["answer"]
|
|
|
|
|
|
def test_all_aspects_absent_is_no_answer_not_rejection(corpus):
|
|
db, raw = corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.'")
|
|
q = "Aké problémy riešili Eva Nová a Boris Nový?"
|
|
r = generate_verified_rag(
|
|
q, prepare(corpus, q), lambda _: pytest.fail("No model for a bounded empty result")
|
|
)
|
|
assert r["status"] == "no_answer" and r["accepted_for_delivery"] and not r["claims"]
|
|
|
|
|
|
def test_known_year_is_returned_when_requested_title_is_missing(corpus):
|
|
_, raw = corpus
|
|
source = raw["sources"][0]
|
|
source["text"] = "Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser."
|
|
raw["sources"] = [source]
|
|
raw["context"] = build_context_text(raw["sources"])
|
|
q = "Uveď názov a rok diplomovej práce Evy Novej."
|
|
a = {
|
|
"status": "ok",
|
|
"answer": "",
|
|
"claims": [{"person": "Eva Nová", "work_type": "diploma_thesis", "year": 2024}],
|
|
"evidence_quotes": [],
|
|
"source_urls": [source["source_url"]],
|
|
}
|
|
r = generate_verified_rag(q, raw, complete(a))
|
|
assert r["accepted_for_delivery"] and "2024" in r["answer"]
|
|
assert "názov práce nie je v poskytnutých podkladoch doložený" in r["answer"]
|
|
assert r["answer_coverage"]["status"] == "partial" and not r["claims"][0].get("title")
|
|
|
|
|
|
def test_concrete_results_win_over_administrative_notes(corpus):
|
|
q = "Aké výsledky dosiahla Eva Nová?"
|
|
p = prepare(corpus, q)
|
|
c = rag_evidence_context(q, p)
|
|
text = "\n".join(x["text"] for x in c["content_candidates"])
|
|
assert "Natrénovaný model BERT" in text
|
|
r = generate_verified_rag(q, p, complete(model_selection(q, p)))
|
|
assert r["accepted_for_delivery"] and "Natrénovaný model BERT" in r["answer"]
|
|
assert "Konzultácia" not in r["answer"]
|
|
|
|
|
|
def test_duplicate_quotes_and_overlapping_chunks_do_not_duplicate_output(corpus):
|
|
q = "Aké výsledky dosiahla Eva Nová?"
|
|
p = prepare(corpus, q)
|
|
a = model_selection(q, p)
|
|
a["evidence_quotes"] *= 2
|
|
r = generate_verified_rag(q, p, complete(a))
|
|
assert r["accepted_for_delivery"] and r["answer"].count("Natrénovaný model BERT") == 1
|
|
assert "Ďalšie podklady:" not in r["answer"] and "Ďalšie doložené podklady" not in r["answer"]
|
|
|
|
|
|
def test_plans_cannot_answer_actual_use_or_completed_experiments():
|
|
assert "methods" not in facets_for(
|
|
"Použite metódu BERT.", "task", "Aké metódy skutočne používal?"
|
|
)
|
|
assert "experiments" not in facets_for(
|
|
"Pripravte experiment.", "task", "Aké experimenty vykonal?"
|
|
)
|
|
assert "results" not in facets_for(
|
|
"Cieľom je implementovať systém.", "progress", "Aké dosiahol výsledky?"
|
|
)
|
|
|
|
|
|
def test_method_answer_does_not_add_valid_but_unrequested_profile_metadata(corpus):
|
|
_, raw = corpus
|
|
raw["sources"][0]["author"] = "Daniel Hladek"
|
|
q = "Aké metódy používala Eva Nová pri diplomovej práci v roku 2024? Uveď aj rok práce."
|
|
p = prepare(corpus, q)
|
|
a = model_selection(q, p)
|
|
a["claims"] = [
|
|
{
|
|
"person": "Eva Nová",
|
|
"work_type": "diploma_thesis",
|
|
"year": 2024,
|
|
"document_author": "Daniel Hladek",
|
|
}
|
|
]
|
|
r = generate_verified_rag(q, p, complete(a))
|
|
assert r["accepted_for_delivery"] and "BERT" in r["answer"]
|
|
assert "Daniel Hladek" not in r["answer"] and "autor dokumentu" not in r["answer"]
|
|
assert r["answer"].count("2024") == 1
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
[
|
|
"Práce od roku 2020 do roku 2024?",
|
|
"Práce pred rokom 2020?",
|
|
"Študenti s aspoň 3 prácami?",
|
|
"Práce okrem diplomových?",
|
|
"Študenti bez tagu RAG?",
|
|
"V ktorom roku Ján Holp začal pracovať na diplomovej práci?",
|
|
"Kedy Ján Holp obhájil diplomovú prácu?",
|
|
],
|
|
)
|
|
def test_unsupported_operators_are_not_ignored(question):
|
|
assert unsupported_constraint(question)
|
|
result = gs.search_graph_with_session(FakeSession({}), question)
|
|
assert (
|
|
result["status"] == "ambiguous"
|
|
and result["reason"] == "unsupported_constraint"
|
|
and not result["facts"]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"question",
|
|
[
|
|
"Práca v roku 1999?",
|
|
"Aký je názov práce „Od 2020 do 2024“?",
|
|
"Aké práce alebo projekty má Eva Nová?",
|
|
],
|
|
)
|
|
def test_supported_year_and_work_alternative_are_not_blocked(question):
|
|
assert unsupported_constraint(question) is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"reason",
|
|
["model_unavailable", "retrieval_unavailable", "answer_rejected", "answer_capacity_exceeded"],
|
|
)
|
|
def test_public_errors_are_distinct_and_do_not_leak_rejected_text(reason):
|
|
r = public_failure(
|
|
{
|
|
"reason": reason,
|
|
"answer": "SECRET YEAR 2099",
|
|
"claims": [{"person": "Eva"}],
|
|
"source_urls": ["SECRET"],
|
|
}
|
|
)
|
|
assert r["reason"] == reason and not r["accepted_for_delivery"]
|
|
assert "SECRET" not in r["answer"] and "2099" not in r["answer"]
|
|
assert not r["claims"] and not r["source_urls"]
|
|
|
|
|
|
def test_retrieval_failure_is_not_misreported_as_bad_model_answer(monkeypatch):
|
|
import tools.answers.answer_delivery as delivery
|
|
|
|
def unavailable(_):
|
|
raise RuntimeError("SECRET backend address")
|
|
|
|
monkeypatch.setattr(delivery, "search_graph", unavailable)
|
|
r = delivery.deliver_answer(
|
|
"Aké práce má Eva Nová?",
|
|
engine="graphrag",
|
|
complete=lambda _: pytest.fail("No model call after failed retrieval"),
|
|
)
|
|
assert r["status"] == "error" and r["reason"] == "retrieval_unavailable"
|
|
assert "SECRET" not in r["answer"]
|