dp-zp-agent/test/test_combined_answers.py
2026-11-16 20:45:10 +01:00

334 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""ZP Wiki + CRZP in one answer: order, labels, clarification, follow-ups, CRZP timeout."""
import time
import pytest
from tools.answers import combined as C
WIKI = {"status": "ok", "reason": None, "accepted_for_delivery": True, "claims": [], "index_version": {"version": "v"},
"complete": True, "truncated": False, "truncation_reasons": [],
"answer": "**Ján Pták — Diplomová práca 2027**\n\nVedúci práce: Daniel Hladek.\n\nZdroje: https://zp.test/jan",
"source_urls": ["https://zp.test/jan"],
"delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": "x"}}
@pytest.fixture
def combo(monkeypatch):
monkeypatch.setenv("SEARCH_API_KEY", "combined-test-key")
monkeypatch.setenv("ANSWER_WITH_CRZP", "1")
state = {"part": None, "match": {"name": "Ján Pták", "student": True}}
monkeypatch.setattr("tools.crzp.answering.wiki_bridge.wiki_match", lambda q, db_file=None: state["match"])
monkeypatch.setattr(C, "crzp_part", lambda q, person, complete=None, student=True: state["part"])
# The composition of these tests, not the records of data/ (tools/answers/unified.py has its own tests).
monkeypatch.setattr("tools.answers.unified.catalog", lambda: {"people": {}, "works": {}})
monkeypatch.setattr("tools.answers.unified.recency", lambda name: "")
monkeypatch.setattr("tools.answers.unified.registry_rows", lambda person: [])
monkeypatch.setenv("NAME_FORMS_MODEL", "0")
monkeypatch.setattr("tools.answers.names.spelled", lambda name: name)
monkeypatch.setattr(C, "_one_person", lambda q: None)
state["deliver"] = lambda q, **kw: dict(WIKI)
return state
def bound(result, question):
"""Like the real wiki answer: bound to the question it was asked."""
from tools.answers.delivery_contract import fingerprint
check = result.get("delivery_check")
if isinstance(check, dict) and check.get("request_fingerprint") == "x":
result = {**result, "delivery_check": {**check, "request_fingerprint": fingerprint(question, "rag")}}
return result
def ask(combo, question, token=None):
return C.answer(question, engine="rag", context_token=token,
deliver=lambda q, **kw: bound(combo["deliver"](q, **kw), q))
def test_wiki_first_then_the_marked_crzp_part(combo):
combo["part"] = {"mode": "person", "text": "- bakalárska práca 2025 „IoT“ — vedúci Pavol Galajda; oponent Daniel Hládek",
"rows": [], "urls": ["https://opac.test/1"], "people": [], "works": []}
result = ask(combo, "Kto vedie Jána Ptáka?")
text = result["answer"]
assert text.index("Vedúci práce: Daniel Hladek") < text.index("Pavol Galajda") and "CRZP" not in text
assert result["source_urls"] == ["https://zp.test/jan", "https://opac.test/1"] and result["status"] == "ok"
assert text.count("Zdroje:") == 1 and text.rstrip().endswith("https://zp.test/jan, https://opac.test/1")
import packages.claude.client as client
client.validate_metadata(result)
assert client.validated_delivery_check(result, "Kto vedie Jána Ptáka?", "rag")
def test_without_crzp_the_wiki_answer_is_unchanged_but_for_wording(combo):
result = ask(combo, "Kedy mal Ján Pták posledné stretnutie?")
assert "V CRZP" not in result["answer"] and result["answer"].startswith("**Ján Pták")
def test_a_work_only_crzp_has_is_asked_for_and_the_choice_continues(combo, monkeypatch):
combo["part"] = {"mode": "person", "text": "x", "rows": [], "urls": [], "people": [],
"works": [{"source": "crzp", "label": "bakalárska práca 2025 „IoT“", "type": "bachelor_thesis", "key": "k"}]}
monkeypatch.setattr(C, "wiki_works", lambda person: {"Diplomová práca 2027": {"type": "diploma_thesis", "topic": "Agent"}})
asked = ask(combo, "Aké technológie využíva Ján Pták vo svojej práci?")
assert asked["status"] == "ambiguous" and "1. Diplomová práca 2027 „Agent“" in asked["answer"]
assert "2. bakalárska práca 2025 „IoT“" in asked["answer"] and "CRZP" not in asked["answer"]
seen = []
combo["deliver"] = lambda q, **kw: (seen.append(q), dict(WIKI))[1]
ask(combo, "1", asked["conversation_context"]["token"])
assert seen == ["Aké technológie využíva Ján Pták vo svojej diplomovej práci?"]
chosen = []
monkeypatch.setattr(C, "_answer_choice", lambda q, c, **kw: chosen.append(c) or dict(WIKI))
ask(combo, "tú bakalársku", asked["conversation_context"]["token"])
assert chosen[0]["source"] == "crzp"
LIST = {**WIKI, "answer": "**Ján Pták**\n\n- Diplomová práca 2027: GraphRAG.\n\n**Maksym Zatirka**\n\n- Bakalárska "
"práca 2026: Neo4J.\n\nZdroje: https://zp.test/students/2022/jan_ptak, https://zp.test/students/2023/maksym_zatirka",
"source_urls": ["https://zp.test/students/2022/jan_ptak", "https://zp.test/students/2023/maksym_zatirka"]}
def crzp_items(count):
return [{"name": f"Autor {chr(65 + i)}", "school": "Slovenská technická univerzita v Bratislave", "source": "crzp",
"text": f"Slovenská technická univerzita v Bratislave: „Práca {i}“ (bakalárska práca, 2025)",
"urls": [f"https://opac.test/{i}"], "sid": f"S{i}", "title": f"Práca {i}"} for i in range(count)]
def test_one_numbered_list_with_sources_at_the_end_and_the_rest_offered(combo):
combo["match"] = None
combo["deliver"] = lambda q, **kw: dict(LIST)
combo["part"] = {"mode": "topic", "items": crzp_items(7), "rows": [], "urls": [], "text": ""}
first = ask(combo, "Kto pracoval s GraphRAG?")
text = first["answer"]
assert "**1. Ján Pták**" in text and "**2. Maksym Zatirka**" in text and "ZP Wiki" not in text
assert "**3. Autor A** — Slovenská technická" in text and "**6. Autor D**" in text and "Autor E" not in text
assert text.index("Zdroje:") < text.index("Našli sa ešte 3 ďalšie osoby. Chceš vidieť ďalšie?")
assert "https://opac.test/3" in text and "https://opac.test/4" not in text
more = ask(combo, "áno", first["conversation_context"]["token"])
assert "**7. Autor E** — " in more["answer"] and "**9. Autor G**" in more["answer"]
assert "Chceš vidieť" not in more["answer"]
import packages.claude.client as client
assert client.validated_delivery_check(more, "áno", "rag")
def test_schools_of_the_people_of_the_previous_answer(combo):
combo["match"] = None
combo["deliver"] = lambda q, **kw: dict(LIST)
combo["part"] = {"mode": "similar", "items": crzp_items(1), "rows": [], "urls": [], "text": ""}
first = ask(combo, "Kto pracoval na podobnej téme ako Ján Pták?")
schools = ask(combo, "a z akých škôl boli tí študenti?", first["conversation_context"]["token"])
assert "1. Ján Pták — Technická univerzita v Košiciach (KEMT FEI)" in schools["answer"]
assert "3. Autor A — Slovenská technická univerzita v Bratislave" in schools["answer"]
def test_an_ordinal_points_at_the_shown_person(combo):
combo["match"] = None
combo["deliver"] = lambda q, **kw: dict(LIST)
combo["part"] = None
first = ask(combo, "Kto pracoval s Neo4j?")
seen = []
combo["deliver"] = lambda q, **kw: (seen.append(q), dict(WIKI))[1]
combo["match"] = {"name": "Maksym Zatirka", "student": True}
result = ask(combo, "a čo robil druhý z nich?", first["conversation_context"]["token"])
assert seen == ["a čo robil Maksym Zatirka?"]
import packages.claude.client as client
assert client.validated_delivery_check(result, "a čo robil druhý z nich?", "rag")
def test_a_foreign_fingerprint_is_never_rebound(combo):
combo["deliver"] = lambda q, **kw: {**WIKI, "delivery_check": {**WIKI["delivery_check"], "request_fingerprint": "0" * 64}}
result = ask(combo, "Kto vedie Jána Ptáka?")
assert result["delivery_check"]["request_fingerprint"] == "0" * 64 # the client rejects it
def test_the_reply_to_a_work_choice_is_bound_to_the_reply(combo, monkeypatch):
combo["part"] = {"mode": "person", "text": "x", "rows": [], "urls": [],
"works": [{"source": "crzp", "label": "bakalárska práca 2025 „IoT“", "type": "bachelor_thesis", "key": "k"}]}
monkeypatch.setattr(C, "wiki_works", lambda person: {"Diplomová práca 2027": {"type": "diploma_thesis", "topic": "Agent"}})
asked = ask(combo, "Aké technológie využíva Ján Pták vo svojej práci?")
token = asked["conversation_context"]["token"]
import packages.claude.client as client
for reply in ("1", "tak 1", "jednotku", "tú diplomovú"):
result = ask(combo, reply, token)
assert result["status"] == "ok" and client.validated_delivery_check(result, reply, "rag"), reply
after = ask(combo, "2", token) # the CRZP work; the offer stays for the other one
assert after["conversation_context"]["token"] and ask(combo, "tak 1", after["conversation_context"]["token"])["status"] == "ok"
combo["match"] = None
again = ask(combo, "neviem", token)
assert again["status"] == "ambiguous" and "1 – Diplomová práca 2027" in again["answer"]
def test_a_slow_crzp_does_not_hold_the_wiki_answer(combo, monkeypatch):
monkeypatch.setattr(C, "CRZP_SECONDS", 0.5)
monkeypatch.setattr(C, "crzp_part", lambda *a, **k: (time.sleep(4), None)[1])
started = time.monotonic()
result = ask(combo, "Kto vedie Jána Ptáka?")
assert result["status"] == "ok" and time.monotonic() - started < 2.0
def test_a_forged_combined_token_is_refused(combo):
first = ask(combo, "Kto vedie Jána Ptáka?")
token = first["conversation_context"]["token"]
assert ask(combo, "a z akých škôl?", token[:-3] + "abc")["reason"] == "invalid_context"
def test_crzp_people_and_keywords_link_to_the_wiki_graph_by_code():
from tools.crzp.storage.graph import link_pairs
rows = [{"school": "Technická univerzita v Košiciach", "authors": ["Pták, Ján"], "supervisors": ["Galajda, Pavol"],
"opponents": [], "keywords": ["Speech Recognition", "iot"]},
{"school": "Iná univerzita", "authors": ["Pták, Jan"], "supervisors": [], "opponents": [], "keywords": []}]
people, keywords = link_pairs(rows, ["Ján Pták", "Daniel Hladek"], ["rozpoznávanie reči", "Dialógový systém"])
assert people == [{"crzp": "Pták, Ján", "wiki": "Ján Pták"}] # only at TUKE, the namesake elsewhere is not linked
assert keywords == [{"key": "speech recognition", "wiki": "rozpoznávanie reči"}] # through the reviewed synonyms
def test_the_wiki_question_which_person_wins_over_crzp(combo):
combo["match"] = None
combo["part"] = {"mode": "explicit", "status": "ambiguous", "text": "Menu „Ján“ zodpovedá v CRZP…", "rows": [],
"urls": []}
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "ambiguous_person",
"answer": "Ktorého Jána myslíš? Ján Holp, Ján Pták."}
result = ask(combo, "Na čom pracoval Ján?")
assert result["answer"] == "Ktorého Jána myslíš? Ján Holp, Ján Pták."
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "missing_subject",
"answer": "Uveď meno študenta."}
combo["part"] = {**combo["part"], "status": "ok", "text": "Školitelia prác …"}
assert "Školitelia prác" in ask(combo, "Kto viedol najviac prác o strojovom učení?")["answer"]
def test_an_unknown_person_goes_to_crzp_only_for_a_thesis_question():
assert C.THESIS.search(C.normalized("O čom je bakalárska práca Jána Nováka?"))
assert not C.THESIS.search(C.normalized("Na čom pracuje Peter Novák?"))
assert not C.ABOUT.search(C.normalized("Čomu sa venuje Jakub Schwarc?"))
def test_a_person_in_both_sources_is_listed_once():
wiki = [{"name": "Jakub Maruniak", "school": C.WIKI_SCHOOL, "source": "wiki", "text": "**Jakub Maruniak**", "urls": []}]
crzp = [{"name": "Jakub Maruniak", "school": C.TUKE, "source": "crzp", "text": f"{C.TUKE}: „Anotácia“ (diplomová práca, 2021)",
"urls": ["https://opac.test/m"]},
{"name": "Eva Malá", "school": "Iná", "source": "crzp", "text": "Iná: „X“", "urls": []}]
rest = C.merge_people(wiki, crzp)
assert [i["name"] for i in rest] == ["Eva Malá"] and "V CRZP: „Anotácia“" in wiki[0]["text"]
assert wiki[0]["urls"] == ["https://opac.test/m"]
def test_a_crzp_work_only_found_through_its_abstract_is_no_topic_answer(monkeypatch):
from tools.crzp.answering.understand import CrzpQuery
monkeypatch.setattr("tools.retrieval.rerank.scores", lambda q, texts: [-3.0] * len(texts))
rows = [{"title": "Fairy Tale Intertexts in Shakespeare's Tragedies", "keywords": ["intertext"]},
{"title": "Anotácia a rozpoznávanie pomenovaných entít", "keywords": ["anotácia"]}]
kept = C.relevant("Kto robil anotácie v Prodigy?", CrzpQuery(topics=["anotácie"]), rows)
assert [r["title"] for r in kept] == ["Anotácia a rozpoznávanie pomenovaných entít"]
def test_related_topics_skip_the_topic_its_synonyms_and_generic_tags(monkeypatch):
from tools.crzp.answering.understand import CrzpQuery
rows = [{"title": f"T{i}", "year": 2020 + i, "authors": ["Malá, Eva"], "school": "Technická univerzita v Košiciach",
"url": f"https://opac.test/{i}",
"keywords": ["rozpoznávanie reči", "speech recognition", "MFCC", "python", "syntéza reči"]} for i in range(3)]
monkeypatch.setattr("tools.crzp.answering.answer.find", lambda query: ({}, rows))
monkeypatch.setattr(C, "tag_frequencies", lambda: ({"mfcc": 20, "synteza reci": 30}, 30000))
monkeypatch.setattr(C, "wiki_people_for", lambda topic: ["Ján Pták"] if topic == "MFCC" else [])
part = C.related_part(CrzpQuery(topics=["rozpoznávanie reči"], intent="keywords"))
names = [i["name"] for i in part["items"]]
assert names == ["MFCC", "Syntéza reči"] and part["topic"] == "rozpoznávanie reči"
assert part["items"][0]["text"].startswith("3 práce, napr. „") and part["items"][0]["text"].endswith("venujú sa mu aj Ján Pták")
def test_synonym_groups_match_case_forms_quickly():
import time as _time
from tools.retrieval.synonyms import base_name, group_of
started = _time.monotonic()
for i in range(20000):
base_name(f"tag {i}")
assert _time.monotonic() - started < 2.0
assert base_name("skrytých Markovových modelov") == "skryté Markovove modely" and base_name("HMM") == base_name("hidden Markov model")
assert group_of("python") == ["python"]
def test_a_crzp_follow_up_is_bound_to_the_engine_asked(combo, monkeypatch):
from tools.crzp.answering import answer as crzp
from tools.answers.delivery_contract import fingerprint
combo["match"] = None
combo["part"] = {"mode": "explicit", "status": "ambiguous", "text": "Ktorú osobu myslíš?\n\n1. A\n2. B",
"rows": [], "urls": [], "crzp_token": "crzp-token"}
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "missing_subject", "answer": "?"}
first = ask(combo, "Kto bol oponent práce Novák?")
monkeypatch.setattr(crzp, "_answer", lambda q, complete=None, context_token=None: {
"status": "ok", "reason": None, "accepted_for_delivery": True, "answer": "Oponent: X\n\nZdroj: https://opac.test/1",
"source_urls": ["https://opac.test/1"], "claims": [], "index_version": None, "complete": True, "truncated": False,
"truncation_reasons": [], "delivery_check": {"layer": "delivery", "code": "complete",
"request_fingerprint": fingerprint(q, "crzp")}})
chosen = ask(combo, "3", first["conversation_context"]["token"])
import packages.claude.client as client
assert chosen["status"] == "ok" and client.validated_delivery_check(chosen, "3", "rag")
def test_other_schools_exclude_tuke_and_drop_the_wiki_part():
from tools.crzp.answering.understand import understand
from tools.crzp.answering.answer import keep
query = understand("Aké práce o detekcii nenávistnej reči sú na iných školách?", use_model=False)
assert query.exclude_school == "tuke" and not {"iny", "skola"} & set(query.topics)
assert understand("Kto robil chatboty mimo TUKE?", use_model=False).exclude_school == "tuke"
assert understand("Kto robil chatboty na STU?", use_model=False).exclude_school is None
tuke = {"title": "A", "school": "Technická univerzita v Košiciach", "year": 2024}
stu = {"title": "B", "school": "Slovenská technická univerzita v Bratislave", "year": 2024}
assert not keep(tuke, query) and keep(stu, query)
def test_bilingual_twin_topics_are_one_related_topic(monkeypatch):
from tools.crzp.answering.understand import CrzpQuery
rows = [{"title": f"T{i}", "year": 2020 + i, "authors": ["Malá, Eva"], "school": "Univerzita X",
"url": f"https://opac.test/{i}", "keywords": ["chatbot", "konverzačný marketing", "conversational marketing"]}
for i in range(3)]
monkeypatch.setattr("tools.crzp.answering.answer.find", lambda query: ({}, rows))
monkeypatch.setattr(C, "tag_frequencies", lambda: ({}, 30000))
monkeypatch.setattr(C, "wiki_people_for", lambda topic: [])
part = C.related_part(CrzpQuery(topics=["chatbot"], intent="keywords"))
assert [i["name"] for i in part["items"]] in (["Konverzačný marketing"], ["Conversational marketing"])
def test_a_crzp_work_not_matching_the_wiki_is_marked_as_a_possible_namesake(monkeypatch):
monkeypatch.setattr(C, "wiki_works", lambda person: {"Bakalárska práca 2024": {"type": "bachelor_thesis",
"topic": "Anotácia a vyhodnotenie slovenskej databázy nenávistnej reči"}})
monkeypatch.setattr(C, "wiki_units", lambda person: [{"source_path": "students/x/README.md", "kind": "topic", "text": "T"}])
monkeypatch.setattr("tools.knowledge.sources.profile_authors", lambda: {"students/x/README.md": "Daniel Hladek"})
assert C.same_person_work("V F", {"title": "Anotácia a vyhodnotenie databázy nenávistnej reči", "supervisors": []})
assert C.same_person_work("V F", {"title": "Iná téma", "supervisors": ["Hládek, Daniel"]})
assert not C.same_person_work("V F", {"title": "Analýza dát pomocou neurónových sietí", "supervisors": ["Ádám, Norbert"]})
def test_one_item_per_person_even_when_the_works_are_listed_apart():
body = "**Andrii Pervashov — Diplomová práca 2027**\n\n- A\n\n**Nikita Bodnar — Bakalárska práca 2025**\n\n- B\n\n" \
"**Andrii Pervashov — Bakalárska práca 2025**\n\n- C"
head, items, tail, general = C.wiki_items(body, [])
assert [i["name"] for i in items] == ["Andrii Pervashov", "Nikita Bodnar"] and "- C" in items[0]["text"]
def test_paging_and_school_replies_do_not_swallow_new_questions():
def pager(q):
low = C.normalized(q)
words = C.re.findall(r"[a-z0-9]+", low)
return bool(C.YES.match(low) or C.NEXT_PAGE.search(low) and words and all(w in C.PAGER_WORDS for w in words))
assert pager("áno") and pager("ukáž ďalších") and pager("daj mi ďalšie osoby") and pager("pokračuj")
assert not pager("a čo robil ďalej?") and not pager("povedz viac o jeho práci") and not pager("Kto ďalší robil RAG?")
assert C.SCHOOLS.search("a z akych skol su") and C.THESIS.search(C.normalized("Ktoré školy majú práce o RAG?"))
assert C.NEW_QUESTION.match(C.normalized("Kto robil RAG?")) and not C.NEW_QUESTION.match("tak 1")
def test_finalize_drops_source_labels_supervisor_lines_and_checkboxes():
from tools.answers.finalize import finalize
out = finalize("**P**\n\n- Téma: Názov bakalárskej práce: Metódy prekladu\n\n"
"- Bakalárska práca 2020: Meno vedúceho: Ing. Daniel Hládek, PhD.\n\n"
"- Bakalárska práca 2025 (2024-10-30): [x] Natrénujte model.\n\n- Vedúci: Matúš Pleva")
assert "- Téma: Metódy prekladu" in out and "Meno vedúceho" not in out and "[x]" not in out
assert "- Vedúci: Matúš Pleva" in out # the answer itself stays
def test_a_short_follow_up_about_another_person_repeats_the_previous_question(monkeypatch):
people = {"Kto by mohol spolupracovať s Jakubom Schwarcom?": {"name": "Jakub Schwarc", "input": "Jakubom Schwarcom"},
"Kto je vedúci práce Jakuba Schwarca?": {"name": "Jakub Schwarc", "input": "Jakuba Schwarca"},
"Jan Ptak": {"name": "Ján Pták", "input": "Jan Ptak"}}
monkeypatch.setattr(C, "_one_person", lambda q: people.get(q))
assert C.ellipsis("a Jan Ptak by mohol?", {"question": "Kto by mohol spolupracovať s Jakubom Schwarcom?"}) == \
"Mohol by Ján Pták spolupracovať s osobou Jakub Schwarc?"
assert C.ellipsis("a Jan Ptak?", {"question": "Kto je vedúci práce Jakuba Schwarca?"}) == "Kto je vedúci práce Ján Pták?"
assert C.ellipsis("a čo výsledky?", {"question": "Kto je vedúci práce Jakuba Schwarca?"}) is None
assert C.ellipsis("Kto je Jan Ptak?", {"question": "Kto je vedúci práce Jakuba Schwarca?"}) is None