377 lines
15 KiB
Python
377 lines
15 KiB
Python
"""Name correction must preserve constraints and fail closed before retrieval."""
|
|
|
|
import copy
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.retrieval.person_names import resolve_person_names, sqlite_people
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
|
|
|
|
def person(name, slug):
|
|
return {
|
|
"kind": "Person",
|
|
"name": name,
|
|
"element_id": slug,
|
|
"source_document": f"pages/students/2022/{slug}/README.md",
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def people():
|
|
return [
|
|
person("Ján Holp", "jan_holp"),
|
|
person("Ondrej Megela", "ondrej_megela"),
|
|
person("Ján Malinovský", "jan_malinovsky"),
|
|
person("Ján Pták", "jan_ptak"),
|
|
person("Martin Šarišský", "martin_sarissky"),
|
|
person("Martin Jančura", "martin_jancura"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mention,name",
|
|
[
|
|
("Jan Holp", "Ján Holp"),
|
|
("JÁN HOLP", "Ján Holp"),
|
|
("Jána Holpa", "Ján Holp"),
|
|
(" Jan Holp", "Ján Holp"),
|
|
("Holp Ján", "Ján Holp"),
|
|
("Holpa Jána", "Ján Holp"),
|
|
("Holp, Jan", "Ján Holp"),
|
|
("Ondreja Megelu", "Ondrej Megela"),
|
|
("Martina Šarišského", "Martin Šarišský"),
|
|
("Jána Malinovského", "Ján Malinovský"),
|
|
("Ján Holpp", "Ján Holp"),
|
|
("Ján Hlop", "Ján Holp"),
|
|
("Ondrej Megla", "Ondrej Megela"),
|
|
("Ján malinkvsý", "Ján Malinovský"),
|
|
("Ján Ptká", "Ján Pták"),
|
|
],
|
|
)
|
|
def test_variants_and_bounded_typos(people, mention, name):
|
|
q = f"Aké práce má {mention} v roku 1999?"
|
|
before = copy.deepcopy(people)
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "ok" and [m["name"] for m in r["matches"]] == [name]
|
|
assert (
|
|
r["resolved_query"]
|
|
== q[: q.index(mention.strip())]
|
|
+ name
|
|
+ q[q.index(mention.strip()) + len(mention.strip()) :]
|
|
)
|
|
assert "v roku 1999?" in r["resolved_query"] and people == before
|
|
|
|
|
|
def test_multiple_people_and_all_non_name_conditions_survive(people):
|
|
q = "Porovnaj Jána Holppa a Ondreja Megelu; nie diplomový projekt, iba práce z roku 2021 s tagom RAG."
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "ok"
|
|
assert (
|
|
r["resolved_query"]
|
|
== "Porovnaj Ján Holp a Ondrej Megela; nie diplomový projekt, iba práce z roku 2021 s tagom RAG."
|
|
)
|
|
assert len(r["notices"]) == 1 and "Ján Holp" in r["notices"][0]
|
|
|
|
|
|
def test_plural_keyword_question_does_not_hide_a_misspelled_person(people):
|
|
q = "Aké spoločné kľúčové slová majú Ján Ptká a Ján Malinovský?"
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "ok"
|
|
assert [m["name"] for m in r["matches"]] == ["Ján Pták", "Ján Malinovský"]
|
|
assert r["resolved_query"] == q.replace("Ptká", "Pták")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"instruction",
|
|
[
|
|
"potom pri každej osobe príslušné práce s typom a názvom, ak je uvedený",
|
|
"potom pri každej osobe relevantné práce",
|
|
"potom pri každom študentovi jednotlivé práce",
|
|
"potom pre obe osoby príslušné práce",
|
|
"potom pri vybraných osobách stručné zhrnutie",
|
|
],
|
|
)
|
|
def test_presentation_instructions_are_not_additional_people(people, instruction):
|
|
q = (
|
|
"V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt? "
|
|
"Najprv uveď spoločné roky, " + instruction + ". Pridaj zdroje."
|
|
)
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "ok"
|
|
assert [m["name"] for m in r["matches"]] == ["Ján Holp", "Ondrej Megela"]
|
|
assert r["resolved_query"] == q and r["notices"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("mention", ["Xaver Neexistujúci", "xaver neexistujuci"])
|
|
def test_generic_person_reference_does_not_hide_another_unknown_subject(people, mention):
|
|
q = f"Porovnaj Ján Holp a {mention}. Pri každej osobe uveď príslušné práce."
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "no_answer" and r["reason"] == "unknown_person"
|
|
assert r["mention"] == mention
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"q",
|
|
[
|
|
"Aké práce má Xaver Neexistujúci?",
|
|
"Aké práce má xaver neexistujuci?",
|
|
"Aký je názov práce xaver neexistujuci?",
|
|
"Porovnaj práce Jána Holpa a Xavera Neexistujúceho.",
|
|
"Porovnaj xaver neexistujuci a jan holp.",
|
|
"Porovnaj jan holp a xaver neexistujuci.",
|
|
"Aké práce má Ján Hxxxx?",
|
|
"Aké práce má Xaver?",
|
|
"Z kandidátov Ján Holp, neznama osoba, Ondrej Megela vyber osoby s prácou v roku 2024.",
|
|
"Z kandidátov Ján Holp a neznama osoba vyber osoby s prácou v roku 2024.",
|
|
],
|
|
)
|
|
def test_unknown_name_cannot_turn_into_global_query(people, q):
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "no_answer" and r["reason"] == "unknown_person" and not r["candidates"]
|
|
session = FakeSession({gs.CATALOG_QUERY: people})
|
|
graph = gs.search_graph_with_session(session, q)
|
|
assert graph["status"] == "no_answer" and graph["facts"] == []
|
|
assert all(statement == gs.CATALOG_QUERY for statement, _ in session.calls)
|
|
|
|
|
|
def test_nearby_catalog_names_require_selection_and_exact_name_wins():
|
|
catalog = [person("Martin Novák", "novak"), person("Martin Novík", "novik")]
|
|
r = resolve_person_names("Aké práce má Martin Novek?", catalog)
|
|
assert r["status"] == "ambiguous"
|
|
assert {c["name"] for c in r["candidates"]} == {"Martin Novák", "Martin Novík"}
|
|
assert (
|
|
resolve_person_names("Aké práce má Martin Novák?", catalog)["matches"][0]["name"]
|
|
== "Martin Novák"
|
|
)
|
|
|
|
|
|
def test_short_first_name_is_not_automatically_assigned(people):
|
|
people = [*people, {"kind": "DocumentAuthor", "name": "Ján Holp"}]
|
|
r = resolve_person_names("Aké práce má Ján?", people)
|
|
assert r["status"] == "ambiguous" and len(r["candidates"]) == 3
|
|
assert all(c["name"].startswith("Ján ") for c in r["candidates"])
|
|
|
|
|
|
def test_full_exact_name_wins_over_another_person_inflection():
|
|
catalog = [person("Ján Novák", "jan"), person("Jana Novák", "jana")]
|
|
r = resolve_person_names("Aké práce má Jana Novák?", catalog)
|
|
assert r["matches"][0]["name"] == "Jana Novák"
|
|
|
|
|
|
def test_competing_valid_inflections_require_selection():
|
|
catalog = [person("Ján Novák", "jan"), person("Jana Novák", "jana")]
|
|
r = resolve_person_names("Aký je názov práce Jana Novaka?", catalog)
|
|
assert r["status"] == "ambiguous"
|
|
assert {c["name"] for c in r["candidates"]} == {"Ján Novák", "Jana Novák"}
|
|
|
|
|
|
def test_rag_evidence_uses_resolved_identity_not_another_name_inflection():
|
|
from tools.answers.rag_evidence import rag_evidence_context
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
catalog = [person("Ján Novák", "jan"), person("Jana Novák", "jana")]
|
|
q = "Aký je názov diplomovej práce Jana Novák?"
|
|
sources = [
|
|
{
|
|
"source_id": f"S{i}",
|
|
"title": row["name"],
|
|
"document_path": row["source_document"],
|
|
"source_url": f"https://example.test/{i}",
|
|
"text": "Sekcia: Diplomová práca 2024\nNázov: Výskum jazyka " + str(i) + ".",
|
|
}
|
|
for i, row in enumerate(catalog, 1)
|
|
]
|
|
payload = {
|
|
"sources": sources,
|
|
"context": build_context_text(sources),
|
|
"name_resolution": resolve_person_names(q, catalog),
|
|
}
|
|
c = rag_evidence_context(payload["name_resolution"]["resolved_query"], payload)
|
|
assert c["requirements"]["people"] == ["Jana Novák"]
|
|
assert {f["person"] for f in c["facts"]} == {"Jana Novák"}
|
|
|
|
|
|
def test_canonicalized_question_keeps_natural_answer_grammar():
|
|
from tools.answers.presentation import fact_sentences
|
|
|
|
claim = {
|
|
"person": "Ján Holp",
|
|
"work_type": "diploma_thesis",
|
|
"title": "Ukážkový výskum",
|
|
"year": 2021,
|
|
}
|
|
text = fact_sentences(claim, "Aký je názov diplomovej práce Ján Holp?")
|
|
assert text.startswith("Ján Holp má diplomovú prácu s názvom") and "2021" in text
|
|
assert fact_sentences(claim, "Aký je názov diplomovej práce Jána Holpa?").startswith(
|
|
"Diplomová práca Jána Holpa"
|
|
)
|
|
|
|
|
|
def test_two_damaged_tokens_are_never_guessed(people):
|
|
assert resolve_person_names("Aké práce má Jaann Hlop?", people)["status"] == "no_answer"
|
|
|
|
|
|
def test_document_author_is_not_implicitly_a_student(people):
|
|
catalog = [*people, {"kind": "DocumentAuthor", "name": "Daniel Hladek"}]
|
|
r = resolve_person_names("Aké práce má Daniel Hladek?", catalog)
|
|
assert r["status"] == "no_answer"
|
|
author = resolve_person_names("Ktoré dokumenty napísal autor Daniel Hladekk?", catalog)
|
|
assert author["status"] == "ok" and author["matches"][0]["name"] == "Daniel Hladek"
|
|
|
|
|
|
def test_duplicate_names_can_be_selected_by_actual_profile_path():
|
|
catalog = [person("Ján Novák", "novak_a"), person("Ján Novák", "novak_b")]
|
|
q = "Aké práce má Ján Novák?"
|
|
assert resolve_person_names(q, catalog)["status"] == "ambiguous"
|
|
selected = resolve_person_names(q + " Profil: " + catalog[1]["source_document"], catalog)
|
|
assert (
|
|
selected["status"] == "ok"
|
|
and selected["matches"][0]["source_document"] == catalog[1]["source_document"]
|
|
)
|
|
|
|
|
|
def test_multiword_name_supports_surname_first():
|
|
r = resolve_person_names(
|
|
"Aké práce má Chis Bogdan Paul?", [person("Bogdan Paul Chiș", "bogdan")]
|
|
)
|
|
assert r["status"] == "ok" and r["matches"][0]["name"] == "Bogdan Paul Chiș"
|
|
|
|
|
|
def test_literal_nonstandard_catalog_name_preserves_numeric_suffix():
|
|
catalog = [person("Martin NaN1", "nan")]
|
|
for mention in ("Martin NaN1", "NaN1 Martin"):
|
|
r = resolve_person_names("Aké práce má " + mention + " v roku 1999?", catalog)
|
|
assert (
|
|
r["status"] == "ok" and r["resolved_query"] == "Aké práce má Martin NaN1 v roku 1999?"
|
|
)
|
|
assert resolve_person_names("Aké práce má Martin NaN2?", catalog)["status"] == "no_answer"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"q",
|
|
[
|
|
"Ktorí študenti majú tag RAG a diplomový projekt v roku 2026?",
|
|
"Aké práce boli v roku 2020?",
|
|
"Vyhľadaj práce s názvom „Ján Holpp“.",
|
|
"Ktorí študenti majú tag Ján Holpp?",
|
|
"Ktorí študenti majú kľúčové slovo Ján Holpp a prácu v roku 2026?",
|
|
],
|
|
)
|
|
def test_global_queries_and_quoted_titles_are_not_corrected_as_people(people, q):
|
|
r = resolve_person_names(q, people)
|
|
assert r["status"] == "ok" and r["resolved_query"] == q and not r["matches"]
|
|
|
|
|
|
def test_malinovsky_sarissky_never_matches_martin_jancura(people):
|
|
r = resolve_person_names("Porovnaj Jána Malinovského a Martina Šarišského.", people)
|
|
assert {m["name"] for m in r["matches"]} == {"Ján Malinovský", "Martin Šarišský"}
|
|
|
|
|
|
@pytest.fixture
|
|
def names_db(tmp_path, people):
|
|
path = tmp_path / "index.sqlite"
|
|
with sqlite3.connect(path) as conn:
|
|
conn.execute("CREATE TABLE documents(path,title,published)")
|
|
for row in people:
|
|
conn.execute(
|
|
"INSERT INTO documents VALUES(?,?,?)", (row["source_document"], row["name"], 1)
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO documents VALUES(?,?,?)",
|
|
("pages/students/2022/hidden/README.md", "Xaver Skrytý", 0),
|
|
)
|
|
return path
|
|
|
|
|
|
def test_catalog_honors_publication_policy(names_db):
|
|
assert not any(p["name"] == "Xaver Skrytý" for p in sqlite_people(names_db))
|
|
assert any(p["name"] == "Xaver Skrytý" for p in sqlite_people(names_db, published_only=False))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"q,status",
|
|
[("Aké práce má Xaver Neexistujúci?", "no_answer"), ("Aké práce má Ján?", "ambiguous")],
|
|
)
|
|
def test_rag_does_not_search_or_generate_for_unresolved_subject(names_db, monkeypatch, q, status):
|
|
from tools.rag import rag_utils
|
|
from tools.answers.answer_delivery import deliver_answer
|
|
|
|
monkeypatch.setattr(
|
|
rag_utils, "search_database", lambda *a, **kw: pytest.fail("Must not search globally")
|
|
)
|
|
r = deliver_answer(
|
|
q, engine="rag", db_file=names_db, complete=lambda _: pytest.fail("Must not call model")
|
|
)
|
|
assert r["status"] == status and r["accepted_for_delivery"] and r["source_urls"] == []
|
|
|
|
|
|
def test_graph_correction_retains_year_type_and_person_ids(people):
|
|
session = FakeSession({gs.CATALOG_QUERY: people})
|
|
q = "Aké diplomové práce má Jan Holpp v roku 1999?"
|
|
r = gs.search_graph_with_session(session, q)
|
|
assert r["query"] == q and "Ján Holp" in r["resolved_query"]
|
|
params = next(
|
|
params for statement, params in session.calls if statement == gs.THESIS_FACTS_QUERY
|
|
)
|
|
assert params["person_ids"] == ["jan_holp"] and params["years"] == [1999]
|
|
assert r["constraints"]["thesis_types"] == ["diploma_thesis"]
|
|
|
|
|
|
def test_both_engines_use_same_resolution_and_notice(names_db, people, monkeypatch):
|
|
from tools.answers import answer_delivery as delivery
|
|
from tools.rag import rag_utils
|
|
|
|
question = "Aké práce má Ján Holpp v roku 1999?"
|
|
resolution = resolve_person_names(question, people)
|
|
payload = {
|
|
"status": "ok",
|
|
"sources": [],
|
|
"context": "",
|
|
"name_resolution": resolution,
|
|
"resolved_query": resolution["resolved_query"],
|
|
"index_version": None,
|
|
}
|
|
checked = {
|
|
"status": "no_answer",
|
|
"reason": "no_matching_facts",
|
|
"answer": "Pre rok 1999 sa údaje nenašli.",
|
|
"source_urls": [],
|
|
"claims": [],
|
|
"accepted_for_delivery": True,
|
|
"complete": True,
|
|
"truncated": False,
|
|
"truncation_reasons": [],
|
|
}
|
|
monkeypatch.setattr(
|
|
rag_utils,
|
|
"search_database",
|
|
lambda *a, **k: {"results": [], "engine": "hybrid", "strategies": []},
|
|
)
|
|
monkeypatch.setattr(delivery, "build_rag_context", lambda *a, **k: payload)
|
|
monkeypatch.setattr(delivery, "expand_comparison_sources", lambda db, q, p: p)
|
|
monkeypatch.setattr(delivery, "prepare_named_content", lambda db, q, p: p)
|
|
monkeypatch.setattr(delivery, "compact_rag_payload", lambda p: p)
|
|
|
|
def rag(q, p, model):
|
|
assert q == resolution["resolved_query"]
|
|
return dict(checked)
|
|
|
|
monkeypatch.setattr(delivery, "generate_verified_rag", rag)
|
|
rag_result = delivery.deliver_answer(question, engine="rag", db_file=names_db)
|
|
monkeypatch.setattr(delivery, "search_graph", lambda *a, **k: payload)
|
|
monkeypatch.setattr(delivery, "graph_context", lambda p: {"name_resolution": resolution})
|
|
|
|
def graph(q, context, prompt, model):
|
|
assert q == resolution["resolved_query"]
|
|
return {"accepted_for_delivery": True, "answer": dict(checked), "attempts": []}
|
|
|
|
monkeypatch.setattr(delivery, "generate_checked_answer", graph)
|
|
graph_result = delivery.deliver_answer(question, engine="graphrag", db_file=names_db)
|
|
assert rag_result["answer"] == graph_result["answer"]
|
|
assert rag_result["answer"].startswith("Rozpoznané meno: **Ján Holp**")
|