254 lines
15 KiB
Python
254 lines
15 KiB
Python
"""ZP Wiki + CRZP in one answer: order, labels, clarification, follow-ups, CRZP timeout."""
|
||
import time
|
||
|
||
import pytest
|
||
|
||
from tools.answers import combined as C
|
||
|
||
WIKI = {"status": "ok", "reason": None, "accepted_for_delivery": True, "claims": [], "index_version": {"version": "v"},
|
||
"complete": True, "truncated": False, "truncation_reasons": [],
|
||
"answer": "**Ján Pták — Diplomová práca 2027**\n\nVedúci práce: Daniel Hladek.\n\nZdroje: https://zp.test/jan",
|
||
"source_urls": ["https://zp.test/jan"],
|
||
"delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": "x"}}
|
||
|
||
|
||
@pytest.fixture
|
||
def combo(monkeypatch):
|
||
monkeypatch.setenv("SEARCH_API_KEY", "combined-test-key")
|
||
monkeypatch.setenv("ANSWER_WITH_CRZP", "1")
|
||
state = {"part": None, "match": {"name": "Ján Pták", "student": True}}
|
||
monkeypatch.setattr("tools.crzp.answering.wiki_bridge.wiki_match", lambda q, db_file=None: state["match"])
|
||
monkeypatch.setattr(C, "crzp_part", lambda q, person, complete=None, student=True: state["part"])
|
||
state["deliver"] = lambda q, **kw: dict(WIKI)
|
||
return state
|
||
|
||
|
||
def bound(result, question):
|
||
"""Like the real wiki answer: bound to the question it was asked."""
|
||
from tools.answers.delivery_contract import fingerprint
|
||
check = result.get("delivery_check")
|
||
if isinstance(check, dict) and check.get("request_fingerprint") == "x":
|
||
result = {**result, "delivery_check": {**check, "request_fingerprint": fingerprint(question, "rag")}}
|
||
return result
|
||
|
||
|
||
def ask(combo, question, token=None):
|
||
return C.answer(question, engine="rag", context_token=token,
|
||
deliver=lambda q, **kw: bound(combo["deliver"](q, **kw), q))
|
||
|
||
|
||
def test_wiki_first_then_the_marked_crzp_part(combo):
|
||
combo["part"] = {"mode": "person", "text": "- bakalárska práca 2025 „IoT“ — vedúci Pavol Galajda; oponent Daniel Hládek",
|
||
"rows": [], "urls": ["https://opac.test/1"], "people": [], "works": []}
|
||
result = ask(combo, "Kto vedie Jána Ptáka?")
|
||
text = result["answer"]
|
||
assert text.index("Vedúci práce: Daniel Hladek") < text.index("V CRZP") < text.index("Pavol Galajda")
|
||
assert result["source_urls"] == ["https://zp.test/jan", "https://opac.test/1"] and result["status"] == "ok"
|
||
assert text.count("Zdroje:") == 1 and text.rstrip().endswith("https://zp.test/jan, https://opac.test/1")
|
||
import packages.claude.client as client
|
||
client.validate_metadata(result)
|
||
assert client.validated_delivery_check(result, "Kto vedie Jána Ptáka?", "rag")
|
||
|
||
|
||
def test_without_crzp_the_wiki_answer_is_unchanged_but_for_wording(combo):
|
||
result = ask(combo, "Kedy mal Ján Pták posledné stretnutie?")
|
||
assert "V CRZP" not in result["answer"] and result["answer"].startswith("**Ján Pták")
|
||
|
||
|
||
def test_a_work_only_crzp_has_is_asked_for_and_the_choice_continues(combo, monkeypatch):
|
||
combo["part"] = {"mode": "person", "text": "x", "rows": [], "urls": [], "people": [],
|
||
"works": [{"source": "crzp", "label": "bakalárska práca 2025 „IoT“", "type": "bachelor_thesis", "key": "k"}]}
|
||
monkeypatch.setattr(C, "wiki_works", lambda person: {"Diplomová práca 2027": {"type": "diploma_thesis", "topic": "Agent"}})
|
||
asked = ask(combo, "Aké technológie využíva Ján Pták vo svojej práci?")
|
||
assert asked["status"] == "ambiguous" and "1. Diplomová práca 2027 „Agent“ (ZP Wiki)" in asked["answer"]
|
||
assert "2. bakalárska práca 2025 „IoT“ (CRZP)" in asked["answer"]
|
||
seen = []
|
||
combo["deliver"] = lambda q, **kw: (seen.append(q), dict(WIKI))[1]
|
||
ask(combo, "1", asked["conversation_context"]["token"])
|
||
assert seen == ["Aké technológie využíva Ján Pták vo svojej diplomovej práci?"]
|
||
chosen = []
|
||
monkeypatch.setattr(C, "_answer_choice", lambda q, c, **kw: chosen.append(c) or dict(WIKI))
|
||
ask(combo, "tú bakalársku", asked["conversation_context"]["token"])
|
||
assert chosen[0]["source"] == "crzp"
|
||
|
||
|
||
LIST = {**WIKI, "answer": "**Ján Pták**\n\n- Diplomová práca 2027: GraphRAG.\n\n**Maksym Zatirka**\n\n- Bakalárska "
|
||
"práca 2026: Neo4J.\n\nZdroje: https://zp.test/students/2022/jan_ptak, https://zp.test/students/2023/maksym_zatirka",
|
||
"source_urls": ["https://zp.test/students/2022/jan_ptak", "https://zp.test/students/2023/maksym_zatirka"]}
|
||
|
||
|
||
def crzp_items(count):
|
||
return [{"name": f"Autor {chr(65 + i)}", "school": "Slovenská technická univerzita v Bratislave", "source": "crzp",
|
||
"text": f"Slovenská technická univerzita v Bratislave: „Práca {i}“ (bakalárska práca, 2025)",
|
||
"urls": [f"https://opac.test/{i}"], "sid": f"S{i}", "title": f"Práca {i}"} for i in range(count)]
|
||
|
||
|
||
def test_one_numbered_list_with_sources_at_the_end_and_the_rest_offered(combo):
|
||
combo["match"] = None
|
||
combo["deliver"] = lambda q, **kw: dict(LIST)
|
||
combo["part"] = {"mode": "topic", "items": crzp_items(7), "rows": [], "urls": [], "text": ""}
|
||
first = ask(combo, "Kto pracoval s GraphRAG?")
|
||
text = first["answer"]
|
||
assert "**1. Ján Pták** (ZP Wiki)" in text and "**2. Maksym Zatirka** (ZP Wiki)" in text
|
||
assert "**3. Autor A** (CRZP) — Slovenská technická" in text and "**6. Autor D**" in text and "Autor E" not in text
|
||
assert text.index("Zdroje:") < text.index("Našli sa ešte 3 ďalšie osoby (3 v CRZP). Chceš vidieť ďalšie?")
|
||
assert "https://opac.test/3" in text and "https://opac.test/4" not in text
|
||
more = ask(combo, "áno", first["conversation_context"]["token"])
|
||
assert "**7. Autor E** (CRZP)" in more["answer"] and "**9. Autor G**" in more["answer"]
|
||
assert "Chceš vidieť" not in more["answer"]
|
||
import packages.claude.client as client
|
||
assert client.validated_delivery_check(more, "áno", "rag")
|
||
|
||
|
||
def test_schools_of_the_people_of_the_previous_answer(combo):
|
||
combo["match"] = None
|
||
combo["deliver"] = lambda q, **kw: dict(LIST)
|
||
combo["part"] = {"mode": "similar", "items": crzp_items(1), "rows": [], "urls": [], "text": ""}
|
||
first = ask(combo, "Kto pracoval na podobnej téme ako Ján Pták?")
|
||
schools = ask(combo, "a z akých škôl boli tí študenti?", first["conversation_context"]["token"])
|
||
assert "1. Ján Pták — Technická univerzita v Košiciach (KEMT FEI)" in schools["answer"]
|
||
assert "3. Autor A — Slovenská technická univerzita v Bratislave" in schools["answer"]
|
||
|
||
|
||
def test_an_ordinal_points_at_the_shown_person(combo):
|
||
combo["match"] = None
|
||
combo["deliver"] = lambda q, **kw: dict(LIST)
|
||
combo["part"] = None
|
||
first = ask(combo, "Kto pracoval s Neo4j?")
|
||
seen = []
|
||
combo["deliver"] = lambda q, **kw: (seen.append(q), dict(WIKI))[1]
|
||
combo["match"] = {"name": "Maksym Zatirka", "student": True}
|
||
result = ask(combo, "a čo robil druhý z nich?", first["conversation_context"]["token"])
|
||
assert seen == ["a čo robil Maksym Zatirka?"]
|
||
import packages.claude.client as client
|
||
assert client.validated_delivery_check(result, "a čo robil druhý z nich?", "rag")
|
||
|
||
|
||
def test_a_foreign_fingerprint_is_never_rebound(combo):
|
||
combo["deliver"] = lambda q, **kw: {**WIKI, "delivery_check": {**WIKI["delivery_check"], "request_fingerprint": "0" * 64}}
|
||
result = ask(combo, "Kto vedie Jána Ptáka?")
|
||
assert result["delivery_check"]["request_fingerprint"] == "0" * 64 # the client rejects it
|
||
|
||
|
||
def test_the_reply_to_a_work_choice_is_bound_to_the_reply(combo, monkeypatch):
|
||
combo["part"] = {"mode": "person", "text": "x", "rows": [], "urls": [],
|
||
"works": [{"source": "crzp", "label": "bakalárska práca 2025 „IoT“", "type": "bachelor_thesis", "key": "k"}]}
|
||
monkeypatch.setattr(C, "wiki_works", lambda person: {"Diplomová práca 2027": {"type": "diploma_thesis", "topic": "Agent"}})
|
||
asked = ask(combo, "Aké technológie využíva Ján Pták vo svojej práci?")
|
||
token = asked["conversation_context"]["token"]
|
||
import packages.claude.client as client
|
||
for reply in ("1", "tak 1", "jednotku", "tú diplomovú"):
|
||
result = ask(combo, reply, token)
|
||
assert result["status"] == "ok" and client.validated_delivery_check(result, reply, "rag"), reply
|
||
after = ask(combo, "2", token) # the CRZP work; the offer stays for the other one
|
||
assert after["conversation_context"]["token"] and ask(combo, "tak 1", after["conversation_context"]["token"])["status"] == "ok"
|
||
combo["match"] = None
|
||
again = ask(combo, "neviem", token)
|
||
assert again["status"] == "ambiguous" and "1 – Diplomová práca 2027" in again["answer"]
|
||
|
||
|
||
def test_a_slow_crzp_does_not_hold_the_wiki_answer(combo, monkeypatch):
|
||
monkeypatch.setattr(C, "CRZP_SECONDS", 0.5)
|
||
monkeypatch.setattr(C, "crzp_part", lambda *a, **k: (time.sleep(4), None)[1])
|
||
started = time.monotonic()
|
||
result = ask(combo, "Kto vedie Jána Ptáka?")
|
||
assert result["status"] == "ok" and time.monotonic() - started < 2.0
|
||
|
||
|
||
def test_a_forged_combined_token_is_refused(combo):
|
||
first = ask(combo, "Kto vedie Jána Ptáka?")
|
||
token = first["conversation_context"]["token"]
|
||
assert ask(combo, "a z akých škôl?", token[:-3] + "abc")["reason"] == "invalid_context"
|
||
|
||
|
||
def test_crzp_people_and_keywords_link_to_the_wiki_graph_by_code():
|
||
from tools.crzp.storage.graph import link_pairs
|
||
rows = [{"school": "Technická univerzita v Košiciach", "authors": ["Pták, Ján"], "supervisors": ["Galajda, Pavol"],
|
||
"opponents": [], "keywords": ["Speech Recognition", "iot"]},
|
||
{"school": "Iná univerzita", "authors": ["Pták, Jan"], "supervisors": [], "opponents": [], "keywords": []}]
|
||
people, keywords = link_pairs(rows, ["Ján Pták", "Daniel Hladek"], ["rozpoznávanie reči", "Dialógový systém"])
|
||
assert people == [{"crzp": "Pták, Ján", "wiki": "Ján Pták"}] # only at TUKE, the namesake elsewhere is not linked
|
||
assert keywords == [{"key": "speech recognition", "wiki": "rozpoznávanie reči"}] # through the reviewed synonyms
|
||
|
||
|
||
def test_the_wiki_question_which_person_wins_over_crzp(combo):
|
||
combo["match"] = None
|
||
combo["part"] = {"mode": "explicit", "status": "ambiguous", "text": "Menu „Ján“ zodpovedá v CRZP…", "rows": [],
|
||
"urls": []}
|
||
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "ambiguous_person",
|
||
"answer": "Ktorého Jána myslíš? Ján Holp, Ján Pták."}
|
||
result = ask(combo, "Na čom pracoval Ján?")
|
||
assert result["answer"] == "Ktorého Jána myslíš? Ján Holp, Ján Pták."
|
||
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "missing_subject",
|
||
"answer": "Uveď meno študenta."}
|
||
combo["part"] = {**combo["part"], "status": "ok", "text": "Školitelia prác …"}
|
||
assert "Školitelia prác" in ask(combo, "Kto viedol najviac prác o strojovom učení?")["answer"]
|
||
|
||
|
||
def test_an_unknown_person_goes_to_crzp_only_for_a_thesis_question():
|
||
assert C.THESIS.search(C.normalized("O čom je bakalárska práca Jána Nováka?"))
|
||
assert not C.THESIS.search(C.normalized("Na čom pracuje Peter Novák?"))
|
||
assert not C.ABOUT.search(C.normalized("Čomu sa venuje Jakub Schwarc?"))
|
||
|
||
|
||
def test_a_person_in_both_sources_is_listed_once():
|
||
wiki = [{"name": "Jakub Maruniak", "school": C.WIKI_SCHOOL, "source": "wiki", "text": "**Jakub Maruniak**", "urls": []}]
|
||
crzp = [{"name": "Jakub Maruniak", "school": C.TUKE, "source": "crzp", "text": f"{C.TUKE}: „Anotácia“ (diplomová práca, 2021)",
|
||
"urls": ["https://opac.test/m"]},
|
||
{"name": "Eva Malá", "school": "Iná", "source": "crzp", "text": "Iná: „X“", "urls": []}]
|
||
rest = C.merge_people(wiki, crzp)
|
||
assert [i["name"] for i in rest] == ["Eva Malá"] and "V CRZP: „Anotácia“" in wiki[0]["text"]
|
||
assert wiki[0]["urls"] == ["https://opac.test/m"]
|
||
|
||
|
||
def test_a_crzp_work_only_found_through_its_abstract_is_no_topic_answer(monkeypatch):
|
||
from tools.crzp.answering.understand import CrzpQuery
|
||
monkeypatch.setattr("tools.retrieval.rerank.scores", lambda q, texts: [-3.0] * len(texts))
|
||
rows = [{"title": "Fairy Tale Intertexts in Shakespeare's Tragedies", "keywords": ["intertext"]},
|
||
{"title": "Anotácia a rozpoznávanie pomenovaných entít", "keywords": ["anotácia"]}]
|
||
kept = C.relevant("Kto robil anotácie v Prodigy?", CrzpQuery(topics=["anotácie"]), rows)
|
||
assert [r["title"] for r in kept] == ["Anotácia a rozpoznávanie pomenovaných entít"]
|
||
|
||
|
||
def test_related_topics_skip_the_topic_its_synonyms_and_generic_tags(monkeypatch):
|
||
from tools.crzp.answering.understand import CrzpQuery
|
||
rows = [{"title": f"T{i}", "year": 2020 + i, "authors": ["Malá, Eva"], "school": "Technická univerzita v Košiciach",
|
||
"url": f"https://opac.test/{i}",
|
||
"keywords": ["rozpoznávanie reči", "speech recognition", "MFCC", "python", "syntéza reči"]} for i in range(3)]
|
||
monkeypatch.setattr("tools.crzp.answering.answer.find", lambda query: ({}, rows))
|
||
monkeypatch.setattr(C, "tag_frequencies", lambda: ({"mfcc": 20, "synteza reci": 30}, 30000))
|
||
monkeypatch.setattr(C, "wiki_people_for", lambda topic: ["Ján Pták"] if topic == "MFCC" else [])
|
||
part = C.related_part(CrzpQuery(topics=["rozpoznávanie reči"], intent="keywords"))
|
||
names = [i["name"] for i in part["items"]]
|
||
assert names == ["MFCC", "Syntéza reči"] and part["topic"] == "rozpoznávanie reči"
|
||
assert part["items"][0]["text"].startswith("ZP Wiki: Ján Pták; CRZP: 3 práce")
|
||
|
||
|
||
def test_synonym_groups_match_case_forms_quickly():
|
||
import time as _time
|
||
from tools.retrieval.synonyms import base_name, group_of
|
||
started = _time.monotonic()
|
||
for i in range(20000):
|
||
base_name(f"tag {i}")
|
||
assert _time.monotonic() - started < 2.0
|
||
assert base_name("skrytých Markovových modelov") == "skryté Markovove modely" and base_name("HMM") == base_name("hidden Markov model")
|
||
assert group_of("python") == ["python"]
|
||
|
||
|
||
def test_a_crzp_follow_up_is_bound_to_the_engine_asked(combo, monkeypatch):
|
||
from tools.crzp.answering import answer as crzp
|
||
from tools.answers.delivery_contract import fingerprint
|
||
combo["match"] = None
|
||
combo["part"] = {"mode": "explicit", "status": "ambiguous", "text": "Ktorú osobu myslíš?\n\n1. A\n2. B",
|
||
"rows": [], "urls": [], "crzp_token": "crzp-token"}
|
||
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "missing_subject", "answer": "?"}
|
||
first = ask(combo, "Kto bol oponent práce Novák?")
|
||
monkeypatch.setattr(crzp, "_answer", lambda q, complete=None, context_token=None: {
|
||
"status": "ok", "reason": None, "accepted_for_delivery": True, "answer": "Oponent: X\n\nZdroj: https://opac.test/1",
|
||
"source_urls": ["https://opac.test/1"], "claims": [], "index_version": None, "complete": True, "truncated": False,
|
||
"truncation_reasons": [], "delivery_check": {"layer": "delivery", "code": "complete",
|
||
"request_fingerprint": fingerprint(q, "crzp")}})
|
||
chosen = ask(combo, "3", first["conversation_context"]["token"])
|
||
import packages.claude.client as client
|
||
assert chosen["status"] == "ok" and client.validated_delivery_check(chosen, "3", "rag")
|