318 lines
14 KiB
Python
318 lines
14 KiB
Python
"""Question understanding on the real published catalog, without any LLM call.
|
|
|
|
These cases come from questions that real users asked in Claude and from a
|
|
data-driven matrix: every student profile x typical question x name form.
|
|
A complete question must not be answered by a request for a name, and a
|
|
person mentioned by the user must not disappear from the plan.
|
|
"""
|
|
from collections import Counter
|
|
from datetime import date
|
|
import unicodedata
|
|
|
|
import pytest
|
|
|
|
from tools.rag.common import DB_FILE
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
not DB_FILE.is_file() or not (DB_FILE.parent / "knowledge/index.json").is_file(),
|
|
reason="Published local index is required",
|
|
)
|
|
|
|
YEAR = date.today().year
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def catalog():
|
|
from tools.answers.question_plan import planning_catalog
|
|
|
|
return planning_catalog(DB_FILE)
|
|
|
|
|
|
def plan(question, catalog):
|
|
from tools.answers.question_plan import build_question_plan
|
|
|
|
return build_question_plan(question, catalog, complete=None)
|
|
|
|
|
|
def people(result):
|
|
return {(p.name, p.role) for p in result.spec.people}
|
|
|
|
|
|
def names(result):
|
|
return {p.name for p in result.spec.people}
|
|
|
|
|
|
# -- names ------------------------------------------------------------------
|
|
|
|
@pytest.mark.parametrize("question,person", [
|
|
("Pracoval Ján Malinovský na znalostnom grafe?", "Ján Malinovský"),
|
|
("Robil Ján Holp niečo s neurónovými sieťami?", "Ján Holp"),
|
|
("Venoval sa Ján Pták chatbotom?", "Ján Pták"),
|
|
("Aký projekt s LLM robil Ján Malinovský?", "Ján Malinovský"),
|
|
("aky projekt s llm robil jan malinovsky?", "Ján Malinovský"),
|
|
("Aký projekt s Slovak Mistral robil Denis Le Thanh?", "Denis Le Thanh"),
|
|
("Aké výsledky dosiahol Holp?", "Ján Holp"),
|
|
("Na čom pracoval malinovsky?", "Ján Malinovský"),
|
|
])
|
|
def test_named_person_is_kept_and_no_name_is_requested(question, person, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.clarification is None, result.clarification
|
|
assert person in names(result)
|
|
|
|
|
|
@pytest.mark.parametrize("question", ["Na čom pracoval Ján?", "Čo robil Ján?", "Ján"])
|
|
def test_ambiguous_given_name_lists_actual_people(question, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.reason == "incomplete_person_name"
|
|
for person in ("Ján Holp", "Ján Malinovský", "Ján Pták"):
|
|
assert person in result.clarification
|
|
# A supervisor is not offered as a student in a question about student work.
|
|
assert "Ján Staš" not in result.clarification
|
|
assert "pages/" not in result.clarification
|
|
|
|
|
|
@pytest.mark.parametrize("question", [
|
|
"Aký projekt s LLM robil Patrik Vrecko?",
|
|
"aký projekt s llm robil patrik vrecko?",
|
|
])
|
|
def test_unknown_person_is_named_in_the_answer(question, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.reason == "unknown_person"
|
|
assert "atrik" in result.clarification and "Vrecko".lower() in result.clarification.lower()
|
|
|
|
|
|
# -- who worked on a topic ----------------------------------------------------
|
|
|
|
@pytest.mark.parametrize("question", [
|
|
"Kto pracoval na chatbotovi?",
|
|
"Ktorí študenti pracovali na chatbotovi?",
|
|
"Akí študenti pracovali na chatbotovi?",
|
|
"Aki studenti pracovali na chatbote?",
|
|
"Ktori studenti robili s LLM?",
|
|
"Na čom pracovali študenti, ktorí robili chatbota?",
|
|
"Vypíš študentov, ktorí pracovali s rozpoznávaním reči.",
|
|
"Daj mi mená študentov, čo robili chatbota.",
|
|
"Ktoré práce sa venovali chatbotom?",
|
|
])
|
|
def test_unknown_students_are_the_result_not_a_missing_input(question, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.clarification is None, result.clarification
|
|
assert result.spec.intent == "discovery"
|
|
assert not result.spec.people
|
|
|
|
|
|
@pytest.mark.parametrize("question", [
|
|
"Pracoval Ján Malinovský na znalostnom grafe?",
|
|
"Venoval sa Ján Pták chatbotom?",
|
|
"Aký projekt s LLM robil Ján Malinovský?",
|
|
"Robil Ján Holp niečo s neurónovými sieťami?",
|
|
])
|
|
def test_person_and_topic_searches_that_persons_evidence(question, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.spec.intent == "discovery"
|
|
assert len(result.spec.people) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("question", [
|
|
"Na čom sa pracovalo od roku 2022 do roku 2024?",
|
|
"Aké témy sa riešili od roku 2022 do roku 2024?",
|
|
])
|
|
def test_period_question_needs_no_person(question, catalog):
|
|
# Topics/works of an interval are the works of those years (one roster).
|
|
result = plan(question, catalog)
|
|
assert result.clarification is None, result.clarification
|
|
assert result.spec.intent == "metadata"
|
|
assert result.spec.years == [2022, 2023, 2024]
|
|
assert result.spec.period and result.spec.period.field == "work_year"
|
|
|
|
|
|
def test_results_in_a_period_stay_dated_records(catalog):
|
|
result = plan("Aké výsledky sa dosiahli od roku 2022 do roku 2024?", catalog)
|
|
assert result.spec.intent == "period" and result.spec.period.field == "record_date"
|
|
|
|
|
|
# -- supervisors and relative time ------------------------------------------------
|
|
|
|
@pytest.mark.parametrize("question,years", [
|
|
("Ktoré práce vedie Daniel Hladek?", []),
|
|
("Ktoré práce vedie Daniel Hládek?", []),
|
|
("ktoré práce vedie daniel hladek?", []),
|
|
("Ktoré práce vedie Hladek?", []),
|
|
("Ktoré práce má pod sebou Daniel Hladek?", []),
|
|
("Ktoré práce má pod sebou daniel hladek?", []),
|
|
("Ktoré práce viedol Daniel Hladek v roku 2024?", [2024]),
|
|
("Ktoré práce vedie Daniel Hladek tento rok?", [YEAR]),
|
|
("Ktoré práce má tohto roku pod sebou Hládek?", [YEAR]),
|
|
("Ktoré práce viedol Daniel Hladek minulý rok?", [YEAR - 1]),
|
|
])
|
|
def test_supervisor_questions(question, years, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.clarification is None, result.clarification
|
|
# The wiki spells this supervisor both with and without the accent.
|
|
assert ("daniel hladek", "supervisor") in {(_plain(n).lower(), r) for n, r in people(result)}
|
|
assert result.spec.intent == "supervisor"
|
|
assert sorted({y for w in result.spec.works for y in w.years} | set(result.spec.years)) == years
|
|
|
|
|
|
# -- questions that already worked must keep working --------------------------------
|
|
|
|
@pytest.mark.parametrize("question,intent", [
|
|
("Kto pracoval na podobnej téme ako Ján Holp?", "similar"),
|
|
("Ako pokročil Ján Malinovský so svojou prácou?", "progress"),
|
|
("Kedy dosiahol Ján Malinovský aký výsledok?", "timeline"),
|
|
("Kto je vedúci práce Jána Holpa?", "advisors"),
|
|
("Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?", "metadata"),
|
|
("Aké bolo zadanie diplomovej práce Jána Holpa?", "facets"),
|
|
])
|
|
def test_existing_routes_are_unchanged(question, intent, catalog):
|
|
result = plan(question, catalog)
|
|
assert result.clarification is None, result.clarification
|
|
assert result.spec.intent == intent
|
|
|
|
|
|
# -- data-driven matrix ---------------------------------------------------------------
|
|
|
|
TEMPLATES = [
|
|
"Na čom pracoval{a} {S}?",
|
|
"Aké bolo zadanie práce, ktorú robil{a} {S}?",
|
|
"Aké výsledky dosiahol{a} {S}?",
|
|
"Kedy dosiahol{a} {S} aký výsledok?",
|
|
"Kto pracoval na podobnej téme ako {S}?",
|
|
"Čo robil{a} {S} v roku 2024?",
|
|
"Aké kľúčové slová má {S}?",
|
|
"Kto viedol prácu, ktorú robil{a} {S}?",
|
|
"Pracoval{a} {S} na téme strojového prekladu?",
|
|
"Aký projekt s LLM robil{a} {S}?",
|
|
"Ako pokročil{a} {S} so svojou prácou?",
|
|
"Zhrň mi jednoducho prácu, ktorú robil{a} {S}.",
|
|
]
|
|
|
|
|
|
def _plain(value):
|
|
return "".join(c for c in unicodedata.normalize("NFD", value) if unicodedata.category(c) != "Mn")
|
|
|
|
|
|
def test_every_student_in_every_name_form(catalog):
|
|
students = sorted({r["name"] for r in catalog if r.get("kind") == "Person"
|
|
and "/students/" in "/" + (r.get("source_document") or "")})
|
|
assert len(students) > 30
|
|
first = Counter(n.split()[0] for n in students)
|
|
# A surname alone is unambiguous only if no other person uses that word
|
|
# as any part of the name ("Tomáš" can be a surname and a given name).
|
|
last = Counter(_plain(part).lower() for n in students for part in n.split())
|
|
last = {n.split()[-1]: last[_plain(n.split()[-1]).lower()] for n in students}
|
|
failures = []
|
|
for student in students:
|
|
if not all(part.isalpha() for part in student.split()):
|
|
continue # Malformed profile titles are a data issue, reported separately.
|
|
feminine = student.split()[-1].endswith(("á", "ová"))
|
|
forms = {"exact": student, "lower": student.lower(), "plain": _plain(student)}
|
|
if last[student.split()[-1]] == 1 and len(student.split()[-1]) >= 4:
|
|
forms["surname"] = student.split()[-1]
|
|
for template in TEMPLATES:
|
|
for form, name in forms.items():
|
|
question = template.format(S=name, a="a" if feminine else "")
|
|
result = plan(question, catalog)
|
|
if result.clarification or student not in names(result):
|
|
failures.append((form, question, result.reason, (result.clarification or "")[:60]))
|
|
if first[student.split()[0]] > 1:
|
|
result = plan(template.format(S=student.split()[0], a="a" if feminine else ""), catalog)
|
|
if result.reason != "incomplete_person_name" or student not in (result.clarification or ""):
|
|
failures.append(("given", template, result.reason, (result.clarification or "")[:60]))
|
|
assert not failures, f"{len(failures)} failures, e.g. {failures[:15]}"
|
|
|
|
|
|
# -- a person choice continues the previous question ----------------------------------
|
|
|
|
def _pending(question, catalog, monkeypatch):
|
|
monkeypatch.setenv("SEARCH_API_KEY", "k" * 40)
|
|
from tools.answers.conversation import issue_context
|
|
|
|
version = {"version": "test-version"}
|
|
result = {"status": "ambiguous", "accepted_for_delivery": True, "index_version": version,
|
|
"answer": plan(question, catalog).clarification}
|
|
return issue_context(question, result, catalog, {"units": []}, version), version
|
|
|
|
|
|
@pytest.mark.parametrize("reply", ["Ján Holp", "jan holp", "toho Holpa", "Myslím Jána Holpa, prosím."])
|
|
def test_person_choice_answers_the_previous_question(reply, catalog, monkeypatch):
|
|
from tools.answers.conversation import resolve_question
|
|
|
|
context, version = _pending("Na čom pracoval Ján?", catalog, monkeypatch)
|
|
assert {"Ján Holp", "Ján Malinovský", "Ján Pták"} <= set(context["person_options"])
|
|
assert context["work_options"] == [] and context["meeting_options"] == []
|
|
resolved = resolve_question(reply, token=context["token"], catalog=catalog,
|
|
index={"units": []}, version=version)
|
|
assert resolved.query == "Na čom pracoval Ján Holp?"
|
|
assert plan(resolved.query, catalog).clarification is None
|
|
|
|
|
|
def test_new_question_after_a_person_choice_stays_independent(catalog, monkeypatch):
|
|
from tools.answers.conversation import resolve_question
|
|
|
|
context, version = _pending("Na čom pracoval Ján?", catalog, monkeypatch)
|
|
question = "Kto pracoval na chatbotovi a čo mu fungovalo?"
|
|
resolved = resolve_question(question, token=context["token"], catalog=catalog,
|
|
index={"units": []}, version=version)
|
|
assert resolved.query == question
|
|
|
|
|
|
def test_unknown_person_choice_is_not_accepted(catalog, monkeypatch):
|
|
from tools.answers.conversation import resolve_question
|
|
|
|
context, version = _pending("Na čom pracoval Ján?", catalog, monkeypatch)
|
|
resolved = resolve_question("Ján Novák", token=context["token"], catalog=catalog,
|
|
index={"units": []}, version=version)
|
|
assert resolved.query == "Ján Novák"
|
|
|
|
|
|
# -- an unclassified paraphrase is not an answer rejection -----------------------------
|
|
|
|
def test_failed_model_plan_asks_a_concrete_question(catalog):
|
|
from tools.answers.question_plan import build_question_plan
|
|
|
|
def broken_model(messages):
|
|
return {"choices": [{"message": {"content": "not json"}}]}
|
|
|
|
result = build_question_plan("Povedz mi niečo zaujímavé o Jánovi Holpovi.", catalog,
|
|
complete=broken_model, verify_complete=broken_model)
|
|
assert result.reason == "unsupported_content_question"
|
|
assert "Ján Holp" in result.clarification and "výsledky" in result.clarification
|
|
assert "Ján Holp" in names(result)
|
|
|
|
|
|
def test_relative_wiki_link_in_a_quote_is_not_an_unsupported_citation():
|
|
import re
|
|
from tools.answers.context_policy import citation_check
|
|
|
|
quote = "- [Question Answering](/topics/question) - interný projekt, https://zp.kemt.fei.tuke.sk/x"
|
|
shown = re.sub(r"\[([^\]\n]+)\]\((?![a-zA-Z][a-zA-Z0-9+.-]*:|www\.)[^)\s]*\)", r"\1", quote)
|
|
assert shown == "- Question Answering - interný projekt, https://zp.kemt.fei.tuke.sk/x"
|
|
assert citation_check({"answer": shown, "source_urls": []}, {"https://zp.kemt.fei.tuke.sk/x"})["citations_ok"]
|
|
assert not citation_check({"answer": quote, "source_urls": []}, {"https://zp.kemt.fei.tuke.sk/x"})["citations_ok"]
|
|
|
|
|
|
@pytest.mark.parametrize("question,years", [
|
|
("Aké témy sa riešili v roku 2024 a akí študenti?", {2024}),
|
|
("Aké témy sa riešili od roku 2022 do roku 2024 a akí študenti?", {2022, 2023, 2024}),
|
|
])
|
|
def test_topics_of_years_are_a_roster_from_unchanged_source_rows(question, years, catalog):
|
|
from tools.knowledge.roster import roster_question, answer_roster
|
|
from tools.knowledge.store import load_index
|
|
from tools.sync.index_state import read_version
|
|
spec = plan(question, catalog).spec
|
|
assert roster_question(spec)
|
|
index = load_index(DB_FILE.parent, read_version(DB_FILE.parent))
|
|
result = answer_roster(spec, index, engine="rag")
|
|
assert result["status"] == "ok" and result["evidence"]
|
|
original = {r["id"]: r for r in index["units"]}
|
|
assert all(r == original[r["id"]] for r in result["evidence"])
|
|
assert {r["work_year"] for r in result["evidence"]} <= years
|
|
assert "## " not in result["answer"]
|
|
|
|
|
|
def test_counts_and_named_people_are_not_rosters(catalog):
|
|
from tools.knowledge.roster import roster_question
|
|
assert not roster_question(plan("Koľko prác je z roku 2024?", catalog).spec)
|
|
assert not roster_question(plan("Aké práce mal Ján Holp v roku 2021?", catalog).spec)
|