306 lines
18 KiB
Python
306 lines
18 KiB
Python
"""Activity layer (who did what with which tool) and the question catalog."""
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.knowledge import activities as A
|
|
from tools.answers import question_catalog as C
|
|
from tools.answers.question_plan import build_question_plan, direct_spec
|
|
|
|
|
|
def unit(uid, person, phase, text, *, kind=None, work="Bakalárska práca 2021", work_id=None, date=None):
|
|
path = f"pages/students/2020/{person.split()[-1].lower()}/README.md"
|
|
return {"id": uid, "person": person, "source_path": path, "source_url": "https://zp/" + path,
|
|
"line": int(uid[1:]) if uid[1:].isdigit() else 1, "work_label": work, "work_type": "bachelor_thesis",
|
|
"work_year": 2021, "work_id": work_id or (person + work), "kind": kind or {"result": "result",
|
|
"progress": "progress", "planned": "plan", "assignment": "assignment"}.get(phase, "context"),
|
|
"phase": phase, "text": text, "section_scope": "work", "actor": person,
|
|
"actor_status": "profile_context", "annotation_status": "knowledge", "record_date": date}
|
|
|
|
|
|
UNITS = [
|
|
unit("u1", "Andrii Pervashov", "result", "Na korpuse funguje rozpoznávanie reči pomocou Whisper."),
|
|
unit("u2", "Eva Nová", "progress", "Rozpoznávanie nenávistnej reči pomocou transformers."),
|
|
unit("u3", "Samuel Horáni", "unknown", "Chatbot pre mesto", kind="topic"),
|
|
unit("u4", "Samuel Horáni", "result", "Vytvorené prepojenie s databázou a RASA."),
|
|
unit("u5", "Ján Pták", "result", "Prečítané články o chatbotoch."),
|
|
unit("u6", "Peter Plán", "planned", "Vytvorte chatbota pre študentov."),
|
|
unit("u7", "Oleh Poiasnik", "result", "Implementovaný RAG systém s LangChain."),
|
|
unit("u8", "Ján Malinovský", "result", "Funguje retrieval augmented generation nad grafom."),
|
|
unit("u9", "Andrii Pervashov", "result", "Bolo urobené dotrénovanie modelu Whisper."),
|
|
]
|
|
RECORDS = [
|
|
{"unit": "u1", "action": "used", "outcome": "works", "topic": {"text": "rozpoznávanie reči", "canonical": "speech recognition"},
|
|
"tools": [{"text": "Whisper", "canonical": "whisper", "type": "ml_model"}]},
|
|
{"unit": "u2", "action": "implemented", "outcome": "unknown", "topic": {"text": "Rozpoznávanie nenávistnej reči",
|
|
"canonical": "hate speech detection"}, "tools": []},
|
|
{"unit": "u3", "action": "implemented", "outcome": "unknown", "topic": {"text": "Chatbot", "canonical": "chatbot"}, "tools": []},
|
|
{"unit": "u4", "action": "implemented", "outcome": "works", "topic": {"text": "prepojenie s databázou",
|
|
"canonical": "database integration"}, "tools": [{"text": "RASA", "canonical": "rasa", "type": "framework"}]},
|
|
{"unit": "u5", "action": "studied", "outcome": "unknown", "topic": {"text": "chatbotoch", "canonical": "chatbots"}, "tools": []},
|
|
{"unit": "u6", "action": "implemented", "outcome": "unknown", "topic": {"text": "chatbota", "canonical": "chatbot"}, "tools": []},
|
|
{"unit": "u7", "action": "implemented", "outcome": "works", "topic": {"text": "RAG", "canonical": "rag"},
|
|
"tools": [{"text": "LangChain", "canonical": "langchain", "type": "library"}]},
|
|
{"unit": "u8", "action": "used", "outcome": "works", "topic": {"text": "retrieval augmented generation",
|
|
"canonical": "retrieval augmented generation"}, "tools": []},
|
|
{"unit": "u9", "action": "fine_tuned", "outcome": "works", "topic": None,
|
|
"tools": [{"text": "Whisper", "canonical": "whisper", "type": "ml_model"}]},
|
|
]
|
|
|
|
|
|
def layer_and_index(monkeypatch):
|
|
import numpy as np
|
|
monkeypatch.setattr(A, "_embed", lambda texts: np.zeros((len(texts), 4), dtype="float32"))
|
|
concepts, by_key = A.unify(RECORDS, None)
|
|
activities = [{"unit": r["unit"], "action": r["action"], "outcome": r["outcome"],
|
|
"topic": by_key.get(r["topic"]["canonical"]) if r["topic"] else None,
|
|
"topic_text": r["topic"]["text"] if r["topic"] else None,
|
|
"tools": [{"concept": by_key[t["canonical"]], "text": t["text"]} for t in r["tools"]]} for r in RECORDS]
|
|
index = {"version": "k1", "index_version": {"version": "v1"}, "units": UNITS}
|
|
ids = sorted(concepts)
|
|
layer = {"schema": A.ACTIVITY_SCHEMA, "extract_version": A.EXTRACT_VERSION, "knowledge_version": "k1",
|
|
"index_version": {"version": "v1"}, "built_at": "now", "concepts": concepts, "activities": activities,
|
|
"concept_vectors": {"ids": [], "shape": [0, 0], "data": __import__("base64").b64encode(
|
|
__import__("zlib").compress(b"")).decode()}}
|
|
return layer, index, by_key
|
|
|
|
|
|
def test_extraction_keeps_only_literal_phrases():
|
|
item = A._lenient_item({"id": "U1", "activities": [
|
|
{"action": "trained", "topic": {"text": "rozpoznávanie reči", "canonical": "Speech Recognition"},
|
|
"tools": [{"text": "Whisper", "canonical": "whisper", "type": "speech"}, {"text": "GPT-9", "canonical": "gpt"}]},
|
|
{"action": "invented", "topic": {"text": "niečo iné", "canonical": "x"}}]})
|
|
acts = A.validated_activities({"text": "Bolo natrénované Rozpoznávanie reči s Whisper."}, item)
|
|
assert acts[0]["topic"] == {"text": "rozpoznávanie reči", "canonical": "speech recognition"}
|
|
assert [t["text"] for t in acts[0]["tools"]] == ["Whisper"] and acts[0]["tools"][0]["type"] == "other"
|
|
assert len(acts) == 1 # the second activity names no literal phrase of the unit
|
|
|
|
|
|
def test_unification_merges_spelling_and_acronyms_not_neighbours(monkeypatch):
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
assert by_key["chatbot"] == by_key["chatbot"] and by_key["rag"] == by_key["retrieval augmented generation"]
|
|
assert by_key["speech recognition"] != by_key["hate speech detection"]
|
|
assert A.canonical_key("Chatbots") == "chatbot"
|
|
|
|
|
|
def test_speech_recognition_does_not_include_hate_speech(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="rozpoznávaniu reči", canonical="speech recognition")
|
|
people = {r["person"] for r in found["rows"]}
|
|
assert people == {"Andrii Pervashov"}
|
|
|
|
|
|
def test_thesis_topic_counts_its_recorded_work_and_reading_is_not_doing(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="chatbota", canonical="chatbot")
|
|
rows = {r["id"] for r in found["rows"]}
|
|
assert {"u3", "u4"} <= rows # Horáni: thesis about a chatbot + its RASA result
|
|
notes = " ".join(found["notes"])
|
|
assert "Samuel Horáni — téma práce" in notes
|
|
assert "Peter Plán — téma je v zadaní" in notes
|
|
assert "Ján Pták — záznamy dokladajú len štúdium" in notes
|
|
order = [r["person"] for r in found["rows"]]
|
|
assert order.index("Samuel Horáni") < order.index("Peter Plán") < order.index("Ján Pták")
|
|
|
|
|
|
def test_rag_acronym_and_long_name_are_one_topic_and_exclusion_applies(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="RAG", canonical="retrieval augmented generation")
|
|
assert {r["person"] for r in found["rows"]} == {"Oleh Poiasnik", "Ján Malinovský"}
|
|
found = A.discover(graph, None, topic="RAG", canonical="rag", excluded=("LangChain",))
|
|
assert {r["person"] for r in found["rows"]} == {"Ján Malinovský"} and found["excluded"] == ["Oleh Poiasnik"]
|
|
|
|
|
|
def test_action_filters_and_generic_topic(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="model", action="fine_tuned")
|
|
assert {r["person"] for r in found["rows"]} == {"Andrii Pervashov"}
|
|
assert A.question_action("Kto dotrénoval Whisper?") == "fine_tuned"
|
|
assert A.question_action("Kto robil chatbota?") == "any"
|
|
assert A.discover(graph, None, topic="kvantová chémia") is None
|
|
|
|
|
|
def test_stale_layer_is_never_served(monkeypatch, tmp_path):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
(tmp_path / "knowledge").mkdir()
|
|
(tmp_path / A.LAYER_FILE).write_text(json.dumps(layer), encoding="utf-8")
|
|
assert A.load_graph(tmp_path, index) is not None
|
|
assert A.load_graph(tmp_path, {**index, "version": "k2"}) is None
|
|
|
|
|
|
def test_catalog_parameters_must_be_literal():
|
|
decision = C.checked(C.CatalogDecision(type="who_did_topic", people=["Eva"], topic="chatbota",
|
|
topic_canonical="Chat-Bot!", exclude="Whisper"), "Kto robil chatbota bez RASA?")
|
|
assert decision.people == [] and decision.topic == "chatbota" and decision.exclude is None
|
|
assert decision.topic_canonical == "chat bot"
|
|
|
|
|
|
CATALOG = [{"kind": "Person", "name": "Maroš Harahus", "source_document": "pages/students/2016/maros_harahus/README.md"},
|
|
{"kind": "Person", "name": "Ján Pták", "source_document": "pages/students/2022/jan_ptak/README.md"}]
|
|
|
|
|
|
def fake(**kwargs):
|
|
return lambda question: C.CatalogDecision(**kwargs)
|
|
|
|
|
|
def test_catalog_route_overrides_rules_and_keeps_literals():
|
|
plan = build_question_plan("Ktorých študentov vedie Maroš Harahus?", CATALOG,
|
|
classify=fake(type="students_of_advisor", people=["Maroš Harahus"]))
|
|
assert plan.spec.intent == "supervisor" and plan.spec.people[0].role == "supervisor"
|
|
plan = build_question_plan("Kto robil chatbota, ale nepoužil RASA?", CATALOG,
|
|
classify=fake(type="who_did_topic", topic="chatbota", topic_canonical="chatbot", exclude="RASA"))
|
|
assert plan.spec.intent == "discovery" and plan.spec.topic == "chatbota"
|
|
assert [(c.kind, c.quote) for c in plan.spec.conditions] == [("exclusion", "rasa")]
|
|
plan = build_question_plan("Čo robil Ján Pták v októbri 2026?", CATALOG,
|
|
classify=fake(type="activity_in_period", people=["Ján Pták"]))
|
|
assert plan.spec.intent == "progress" and plan.spec.period.start == "2026-10-01"
|
|
|
|
|
|
def test_catalog_failure_or_invalid_route_keeps_rule_plan():
|
|
rules = build_question_plan("Kto je vedúci práce Ján Pták?", CATALOG)
|
|
for classify in (lambda q: None, fake(type="unclear"), fake(type="compare_students", people=["Ján Pták"])):
|
|
assert build_question_plan("Kto je vedúci práce Ján Pták?", CATALOG, classify=classify).spec == rules.spec
|
|
|
|
|
|
def test_out_of_scope_refused_only_without_wiki_subject():
|
|
off = build_question_plan("Aké bude zajtra počasie v Košiciach?", CATALOG, classify=fake(type="out_of_scope"))
|
|
assert off.reason == "request_out_of_scope"
|
|
wiki = build_question_plan("Ktoré témy riešili študenti?", CATALOG, classify=fake(type="out_of_scope"))
|
|
assert wiki.reason != "request_out_of_scope"
|
|
|
|
|
|
def test_runtime_prompt_contains_every_type_and_demos():
|
|
program = {"instructions": C.INSTRUCTIONS, "demos": [{"question": "Kto robil chatbota?",
|
|
"decision": {"type": "who_did_topic", "topic": "chatbota"}}]}
|
|
messages = C.messages_for("Kto robil RAG?", program)
|
|
assert all(t in messages[0]["content"] for t in C.TYPE_DESCRIPTIONS)
|
|
assert [m["role"] for m in messages] == ["system", "user", "assistant", "user"]
|
|
reply = {"choices": [{"message": {"content": json.dumps({"type": "who_did_topic", "topic": "RAG"})}}]}
|
|
C._cache.clear()
|
|
assert C.classify("Kto robil RAG?", lambda m: reply, program=program).topic == "RAG"
|
|
|
|
|
|
def test_narrower_names_need_the_models_decision(monkeypatch):
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
hate = by_key["hate speech detection"]
|
|
graph.concepts[hate]["canonicals"].append("hate speech recognition")
|
|
asked = []
|
|
|
|
def model(messages):
|
|
payload = json.loads(messages[-1]["content"])
|
|
asked.append([c["name"] for c in payload["candidates"]])
|
|
return {"choices": [{"message": {"content": json.dumps({"decisions": [
|
|
{"id": c["id"], "decision": "different"} for c in payload["candidates"]]})}}]}
|
|
found, how = graph.resolve("rozpoznávanie reči", "speech recognition", complete=model)
|
|
assert asked and "hate speech detection" in asked[0] # a candidate, read by the model
|
|
assert found == [by_key["speech recognition"]] and how == "same"
|
|
found, _ = graph.resolve("rozpoznávanie reči", "speech recognition")
|
|
assert found == [by_key["speech recognition"]]
|
|
|
|
|
|
def test_similarity_weights_own_work_and_skips_common_concepts(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
pervashov = {u["source_path"] for u in UNITS if u["person"] == "Andrii Pervashov"}
|
|
assert A.similar_people(graph, pervashov) == [] # nobody else shares Whisper/speech recognition
|
|
poiasnik = {u["source_path"] for u in UNITS if u["person"] == "Oleh Poiasnik"}
|
|
found = A.similar_people(graph, poiasnik)
|
|
assert [f["person"] for f in found] == ["Ján Malinovský"]
|
|
|
|
|
|
def test_exclusion_needs_a_negation_cue_and_generic_topic_is_dropped():
|
|
q = "Kto robil chatbota v RASA v roku 2021?"
|
|
decision = C.checked(C.CatalogDecision(type="who_did_topic", topic="chatbota v RASA", exclude="RASA"), q)
|
|
assert decision.exclude is None
|
|
decision = C.checked(C.CatalogDecision(type="who_did_topic", topic="chatbota", exclude="RASA"),
|
|
"Kto robil chatbota, ale nepoužil RASA?")
|
|
assert decision.exclude == "RASA"
|
|
decision = C.checked(C.CatalogDecision(type="work_status", people=["Samuela Vasiľa"], topic="práca"),
|
|
"V akom stave je práca Samuela Vasiľa?")
|
|
assert decision.topic is None
|
|
route = C.route_for(C.CatalogDecision(type="work_details", people=["X"], topic="RASA"),
|
|
{"matches": [{"name": "X", "source_document": "p"}]})
|
|
assert route.topic is None # only topic-search routes carry a topic
|
|
|
|
|
|
def test_with_an_exact_match_broader_picks_of_the_model_are_ignored(monkeypatch):
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
|
|
def model(messages): # a model that wrongly accepts everything as a kind of the topic
|
|
payload = json.loads(messages[-1]["content"])
|
|
return {"choices": [{"message": {"content": json.dumps({"decisions": [
|
|
{"id": c["id"], "decision": "kind_of"} for c in payload["candidates"]]})}}]}
|
|
found, _ = graph.resolve("Whisper", "whisper", complete=model)
|
|
assert found == [by_key["whisper"]] # no unrelated topic sneaks in beside the exact match
|
|
|
|
|
|
def test_concept_labels_in_notes_carry_no_links(monkeypatch):
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
gid = by_key["chatbot"]
|
|
layer["concepts"][gid]["literals"] = ["chatbota z repozitára https://git.example/x/Rasa", "Chatbot"]
|
|
graph = A.ActivityGraph(layer, index)
|
|
assert "http" not in graph.concept_label(gid) and graph.concept_label(gid)
|
|
|
|
|
|
def test_lemma_keys_and_reranked_record_choice(monkeypatch):
|
|
from tools.retrieval import lemmatize, rerank
|
|
lemmas = {"chatbotom": "chatbot", "chatbota": "chatbot", "chatbotoch": "chatbot", "chatbot": "chatbot"}
|
|
monkeypatch.setattr(lemmatize, "available", lambda: True)
|
|
monkeypatch.setattr(lemmatize, "lemmas", lambda text: [lemmas.get(w, w) for w in normalized_words(text)])
|
|
monkeypatch.setattr(lemmatize.LemmaCache, "many", lambda self, texts: [" ".join(lemmatize.lemmas(t)) for t in texts])
|
|
A._lemma_keys.clear()
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found, how = graph.resolve("chatbotom", None)
|
|
assert by_key["chatbot"] in found and how == "same"
|
|
# The reranker decides which record of a student is shown first.
|
|
monkeypatch.setattr(rerank, "scores", lambda q, texts: [1.0 if "dotrénovanie" in t else 0.0 for t in texts])
|
|
rows = A.discover(graph, None, topic="Whisper", canonical="whisper", question="Kto dotrénoval Whisper?")["rows"]
|
|
assert rows[0]["id"] == "u9"
|
|
A._lemma_keys.clear()
|
|
|
|
|
|
def normalized_words(text):
|
|
import re
|
|
from tools.answers.context_policy import normalized
|
|
return re.findall(r"[a-z0-9+#]+", normalized(text))
|
|
|
|
|
|
def test_fallback_topic_keeps_inflected_topic_words():
|
|
assert A.topic_of_question("Kto robil chatbota?") == "chatbota"
|
|
assert A.topic_of_question("Kto robil chatbota bez RASA?") == "chatbota"
|
|
assert "LLM" in A.topic_of_question("Ktorí študenti robili s LLM a čo im fungovalo?")
|
|
|
|
|
|
def test_advisor_question_keeps_the_named_profile_a_student():
|
|
plan = build_question_plan("Kto vedie Jána Ptáka?", CATALOG,
|
|
classify=fake(type="advisor_of_student", people=["Jána Ptáka"]))
|
|
assert plan.spec.intent == "advisors" and [p.role for p in plan.spec.people] == ["student"]
|
|
|
|
|
|
def test_negation_without_a_topic_is_not_routed_as_exclusion():
|
|
decision = C.CatalogDecision(type="who_did_topic", topic="Python", exclude="Python")
|
|
assert C.route_for(decision, {"matches": []}) is None
|
|
|
|
|
|
def test_literal_safety_net_and_other_activity_bucket(monkeypatch):
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
extra = unit("u10", "Eva Nová", "result", "Je urobené vyhodnotenie WER na korpuse.")
|
|
index["units"].append(extra)
|
|
graph = A.ActivityGraph(layer, index)
|
|
# "WER" was never extracted as a tool; a performed record naming it still counts.
|
|
gid = by_key["whisper"]
|
|
graph.canonical["wer"] = gid # resolve to some concept so the graph path runs
|
|
found = A.discover(graph, None, topic="WER", canonical="wer", action="evaluated")
|
|
assert "u10" in {r["id"] for r in found["rows"]}
|
|
# Recorded work of another kind is not labelled as a mere plan.
|
|
found = A.discover(graph, None, topic="RAG", canonical="rag", action="annotated")
|
|
notes = " ".join(found["notes"])
|
|
assert "nie však priamo požadovanú činnosť" in notes and "téma je v zadaní" not in notes
|