"""Activity layer (who did what with which tool) and the question catalog.""" import json import pytest from tools.knowledge import activities as A from tools.answers import question_catalog as C from tools.answers.question_plan import build_question_plan, direct_spec def unit(uid, person, phase, text, *, kind=None, work="Bakalárska práca 2021", work_id=None, date=None): path = f"pages/students/2020/{person.split()[-1].lower()}/README.md" return {"id": uid, "person": person, "source_path": path, "source_url": "https://zp/" + path, "line": int(uid[1:]) if uid[1:].isdigit() else 1, "work_label": work, "work_type": "bachelor_thesis", "work_year": 2021, "work_id": work_id or (person + work), "kind": kind or {"result": "result", "progress": "progress", "planned": "plan", "assignment": "assignment"}.get(phase, "context"), "phase": phase, "text": text, "section_scope": "work", "actor": person, "actor_status": "profile_context", "annotation_status": "knowledge", "record_date": date} UNITS = [ unit("u1", "Andrii Pervashov", "result", "Na korpuse funguje rozpoznávanie reči pomocou Whisper."), unit("u2", "Eva Nová", "progress", "Rozpoznávanie nenávistnej reči pomocou transformers."), unit("u3", "Samuel Horáni", "unknown", "Chatbot pre mesto", kind="topic"), unit("u4", "Samuel Horáni", "result", "Vytvorené prepojenie s databázou a RASA."), unit("u5", "Ján Pták", "result", "Prečítané články o chatbotoch."), unit("u6", "Peter Plán", "planned", "Vytvorte chatbota pre študentov."), unit("u7", "Oleh Poiasnik", "result", "Implementovaný RAG systém s LangChain."), unit("u8", "Ján Malinovský", "result", "Funguje retrieval augmented generation nad grafom."), unit("u9", "Andrii Pervashov", "result", "Bolo urobené dotrénovanie modelu Whisper."), ] RECORDS = [ {"unit": "u1", "action": "used", "outcome": "works", "topic": {"text": "rozpoznávanie reči", "canonical": "speech recognition"}, "tools": [{"text": "Whisper", "canonical": "whisper", "type": "ml_model"}]}, {"unit": "u2", "action": "implemented", "outcome": "unknown", "topic": {"text": "Rozpoznávanie nenávistnej reči", "canonical": "hate speech detection"}, "tools": []}, {"unit": "u3", "action": "implemented", "outcome": "unknown", "topic": {"text": "Chatbot", "canonical": "chatbot"}, "tools": []}, {"unit": "u4", "action": "implemented", "outcome": "works", "topic": {"text": "prepojenie s databázou", "canonical": "database integration"}, "tools": [{"text": "RASA", "canonical": "rasa", "type": "framework"}]}, {"unit": "u5", "action": "studied", "outcome": "unknown", "topic": {"text": "chatbotoch", "canonical": "chatbots"}, "tools": []}, {"unit": "u6", "action": "implemented", "outcome": "unknown", "topic": {"text": "chatbota", "canonical": "chatbot"}, "tools": []}, {"unit": "u7", "action": "implemented", "outcome": "works", "topic": {"text": "RAG", "canonical": "rag"}, "tools": [{"text": "LangChain", "canonical": "langchain", "type": "library"}]}, {"unit": "u8", "action": "used", "outcome": "works", "topic": {"text": "retrieval augmented generation", "canonical": "retrieval augmented generation"}, "tools": []}, {"unit": "u9", "action": "fine_tuned", "outcome": "works", "topic": None, "tools": [{"text": "Whisper", "canonical": "whisper", "type": "ml_model"}]}, ] def layer_and_index(monkeypatch): import numpy as np monkeypatch.setattr(A, "_embed", lambda texts: np.zeros((len(texts), 4), dtype="float32")) concepts, by_key = A.unify(RECORDS, None) activities = [{"unit": r["unit"], "action": r["action"], "outcome": r["outcome"], "topic": by_key.get(r["topic"]["canonical"]) if r["topic"] else None, "topic_text": r["topic"]["text"] if r["topic"] else None, "tools": [{"concept": by_key[t["canonical"]], "text": t["text"]} for t in r["tools"]]} for r in RECORDS] index = {"version": "k1", "index_version": {"version": "v1"}, "units": UNITS} ids = sorted(concepts) layer = {"schema": A.ACTIVITY_SCHEMA, "extract_version": A.EXTRACT_VERSION, "knowledge_version": "k1", "index_version": {"version": "v1"}, "built_at": "now", "concepts": concepts, "activities": activities, "concept_vectors": {"ids": [], "shape": [0, 0], "data": __import__("base64").b64encode( __import__("zlib").compress(b"")).decode()}} return layer, index, by_key def test_extraction_keeps_only_literal_phrases(): item = A._lenient_item({"id": "U1", "activities": [ {"action": "trained", "topic": {"text": "rozpoznávanie reči", "canonical": "Speech Recognition"}, "tools": [{"text": "Whisper", "canonical": "whisper", "type": "speech"}, {"text": "GPT-9", "canonical": "gpt"}]}, {"action": "invented", "topic": {"text": "niečo iné", "canonical": "x"}}]}) acts = A.validated_activities({"text": "Bolo natrénované Rozpoznávanie reči s Whisper."}, item) assert acts[0]["topic"] == {"text": "rozpoznávanie reči", "canonical": "speech recognition"} assert [t["text"] for t in acts[0]["tools"]] == ["Whisper"] and acts[0]["tools"][0]["type"] == "other" assert len(acts) == 1 # the second activity names no literal phrase of the unit def test_unification_merges_spelling_and_acronyms_not_neighbours(monkeypatch): layer, index, by_key = layer_and_index(monkeypatch) assert by_key["chatbot"] == by_key["chatbot"] and by_key["rag"] == by_key["retrieval augmented generation"] assert by_key["speech recognition"] != by_key["hate speech detection"] assert A.canonical_key("Chatbots") == "chatbot" def test_speech_recognition_does_not_include_hate_speech(monkeypatch): layer, index, _ = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) found = A.discover(graph, None, topic="rozpoznávaniu reči", canonical="speech recognition") people = {r["person"] for r in found["rows"]} assert people == {"Andrii Pervashov"} def test_thesis_topic_counts_its_recorded_work_and_reading_is_not_doing(monkeypatch): layer, index, _ = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) found = A.discover(graph, None, topic="chatbota", canonical="chatbot") rows = {r["id"] for r in found["rows"]} assert {"u3", "u4"} <= rows # Horáni: thesis about a chatbot + its RASA result notes = " ".join(found["notes"]) assert "Samuel Horáni — téma práce" in notes assert "Peter Plán — téma je v zadaní" in notes assert "Ján Pták — záznamy dokladajú len štúdium" in notes order = [r["person"] for r in found["rows"]] assert order.index("Samuel Horáni") < order.index("Peter Plán") < order.index("Ján Pták") def test_rag_acronym_and_long_name_are_one_topic_and_exclusion_applies(monkeypatch): layer, index, _ = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) found = A.discover(graph, None, topic="RAG", canonical="retrieval augmented generation") assert {r["person"] for r in found["rows"]} == {"Oleh Poiasnik", "Ján Malinovský"} found = A.discover(graph, None, topic="RAG", canonical="rag", excluded=("LangChain",)) assert {r["person"] for r in found["rows"]} == {"Ján Malinovský"} and found["excluded"] == ["Oleh Poiasnik"] def test_action_filters_and_generic_topic(monkeypatch): layer, index, _ = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) found = A.discover(graph, None, topic="model", action="fine_tuned") assert {r["person"] for r in found["rows"]} == {"Andrii Pervashov"} assert A.question_action("Kto dotrénoval Whisper?") == "fine_tuned" assert A.question_action("Kto robil chatbota?") == "any" assert A.discover(graph, None, topic="kvantová chémia") is None def test_stale_layer_is_never_served(monkeypatch, tmp_path): layer, index, _ = layer_and_index(monkeypatch) (tmp_path / "knowledge").mkdir() (tmp_path / A.LAYER_FILE).write_text(json.dumps(layer), encoding="utf-8") assert A.load_graph(tmp_path, index) is not None assert A.load_graph(tmp_path, {**index, "version": "k2"}) is None def test_catalog_parameters_must_be_literal(): decision = C.checked(C.CatalogDecision(type="who_did_topic", people=["Eva"], topic="chatbota", topic_canonical="Chat-Bot!", exclude="Whisper"), "Kto robil chatbota bez RASA?") assert decision.people == [] and decision.topic == "chatbota" and decision.exclude is None assert decision.topic_canonical == "chat bot" CATALOG = [{"kind": "Person", "name": "Maroš Harahus", "source_document": "pages/students/2016/maros_harahus/README.md"}, {"kind": "Person", "name": "Ján Pták", "source_document": "pages/students/2022/jan_ptak/README.md"}] def fake(**kwargs): return lambda question: C.CatalogDecision(**kwargs) def test_catalog_route_overrides_rules_and_keeps_literals(): plan = build_question_plan("Ktorých študentov vedie Maroš Harahus?", CATALOG, classify=fake(type="students_of_advisor", people=["Maroš Harahus"])) assert plan.spec.intent == "supervisor" and plan.spec.people[0].role == "supervisor" plan = build_question_plan("Kto robil chatbota, ale nepoužil RASA?", CATALOG, classify=fake(type="who_did_topic", topic="chatbota", topic_canonical="chatbot", exclude="RASA")) assert plan.spec.intent == "discovery" and plan.spec.topic == "chatbota" assert [(c.kind, c.quote) for c in plan.spec.conditions] == [("exclusion", "rasa")] plan = build_question_plan("Čo robil Ján Pták v októbri 2026?", CATALOG, classify=fake(type="activity_in_period", people=["Ján Pták"])) assert plan.spec.intent == "progress" and plan.spec.period.start == "2026-10-01" def test_catalog_failure_or_invalid_route_keeps_rule_plan(): rules = build_question_plan("Kto je vedúci práce Ján Pták?", CATALOG) for classify in (lambda q: None, fake(type="unclear"), fake(type="compare_students", people=["Ján Pták"])): assert build_question_plan("Kto je vedúci práce Ján Pták?", CATALOG, classify=classify).spec == rules.spec def test_out_of_scope_refused_only_without_wiki_subject(): off = build_question_plan("Aké bude zajtra počasie v Košiciach?", CATALOG, classify=fake(type="out_of_scope")) assert off.reason == "request_out_of_scope" wiki = build_question_plan("Ktoré témy riešili študenti?", CATALOG, classify=fake(type="out_of_scope")) assert wiki.reason != "request_out_of_scope" def test_runtime_prompt_contains_every_type_and_demos(): program = {"instructions": C.INSTRUCTIONS, "demos": [{"question": "Kto robil chatbota?", "decision": {"type": "who_did_topic", "topic": "chatbota"}}]} messages = C.messages_for("Kto robil RAG?", program) assert all(t in messages[0]["content"] for t in C.TYPE_DESCRIPTIONS) assert [m["role"] for m in messages] == ["system", "user", "assistant", "user"] reply = {"choices": [{"message": {"content": json.dumps({"type": "who_did_topic", "topic": "RAG"})}}]} C._cache.clear() assert C.classify("Kto robil RAG?", lambda m: reply, program=program).topic == "RAG" def test_narrower_names_need_the_models_decision(monkeypatch): layer, index, by_key = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) hate = by_key["hate speech detection"] graph.concepts[hate]["canonicals"].append("hate speech recognition") asked = [] def model(messages): payload = json.loads(messages[-1]["content"]) asked.append([c["name"] for c in payload["candidates"]]) return {"choices": [{"message": {"content": json.dumps({"decisions": [ {"id": c["id"], "decision": "different"} for c in payload["candidates"]]})}}]} found, how = graph.resolve("rozpoznávanie reči", "speech recognition", complete=model) assert asked and "hate speech detection" in asked[0] # a candidate, read by the model assert found == [by_key["speech recognition"]] and how == "same" found, _ = graph.resolve("rozpoznávanie reči", "speech recognition") assert found == [by_key["speech recognition"]] def test_similarity_weights_own_work_and_skips_common_concepts(monkeypatch): layer, index, _ = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) pervashov = {u["source_path"] for u in UNITS if u["person"] == "Andrii Pervashov"} assert A.similar_people(graph, pervashov) == [] # nobody else shares Whisper/speech recognition poiasnik = {u["source_path"] for u in UNITS if u["person"] == "Oleh Poiasnik"} found = A.similar_people(graph, poiasnik) assert [f["person"] for f in found] == ["Ján Malinovský"] def test_exclusion_needs_a_negation_cue_and_generic_topic_is_dropped(): q = "Kto robil chatbota v RASA v roku 2021?" decision = C.checked(C.CatalogDecision(type="who_did_topic", topic="chatbota v RASA", exclude="RASA"), q) assert decision.exclude is None decision = C.checked(C.CatalogDecision(type="who_did_topic", topic="chatbota", exclude="RASA"), "Kto robil chatbota, ale nepoužil RASA?") assert decision.exclude == "RASA" decision = C.checked(C.CatalogDecision(type="work_status", people=["Samuela Vasiľa"], topic="práca"), "V akom stave je práca Samuela Vasiľa?") assert decision.topic is None route = C.route_for(C.CatalogDecision(type="work_details", people=["X"], topic="RASA"), {"matches": [{"name": "X", "source_document": "p"}]}) assert route.topic is None # only topic-search routes carry a topic def test_with_an_exact_match_broader_picks_of_the_model_are_ignored(monkeypatch): layer, index, by_key = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) def model(messages): # a model that wrongly accepts everything as a kind of the topic payload = json.loads(messages[-1]["content"]) return {"choices": [{"message": {"content": json.dumps({"decisions": [ {"id": c["id"], "decision": "kind_of"} for c in payload["candidates"]]})}}]} found, _ = graph.resolve("Whisper", "whisper", complete=model) assert found == [by_key["whisper"]] # no unrelated topic sneaks in beside the exact match def test_concept_labels_in_notes_carry_no_links(monkeypatch): layer, index, by_key = layer_and_index(monkeypatch) gid = by_key["chatbot"] layer["concepts"][gid]["literals"] = ["chatbota z repozitára https://git.example/x/Rasa", "Chatbot"] graph = A.ActivityGraph(layer, index) assert "http" not in graph.concept_label(gid) and graph.concept_label(gid) def test_lemma_keys_and_reranked_record_choice(monkeypatch): from tools.retrieval import lemmatize, rerank lemmas = {"chatbotom": "chatbot", "chatbota": "chatbot", "chatbotoch": "chatbot", "chatbot": "chatbot"} monkeypatch.setattr(lemmatize, "available", lambda: True) monkeypatch.setattr(lemmatize, "lemmas", lambda text: [lemmas.get(w, w) for w in normalized_words(text)]) monkeypatch.setattr(lemmatize.LemmaCache, "many", lambda self, texts: [" ".join(lemmatize.lemmas(t)) for t in texts]) A._lemma_keys.clear() layer, index, by_key = layer_and_index(monkeypatch) graph = A.ActivityGraph(layer, index) found, how = graph.resolve("chatbotom", None) assert by_key["chatbot"] in found and how == "same" # The reranker decides which record of a student is shown first. monkeypatch.setattr(rerank, "scores", lambda q, texts: [1.0 if "dotrénovanie" in t else 0.0 for t in texts]) rows = A.discover(graph, None, topic="Whisper", canonical="whisper", question="Kto dotrénoval Whisper?")["rows"] assert rows[0]["id"] == "u9" A._lemma_keys.clear() def normalized_words(text): import re from tools.answers.context_policy import normalized return re.findall(r"[a-z0-9+#]+", normalized(text)) def test_fallback_topic_keeps_inflected_topic_words(): assert A.topic_of_question("Kto robil chatbota?") == "chatbota" assert A.topic_of_question("Kto robil chatbota bez RASA?") == "chatbota" assert "LLM" in A.topic_of_question("Ktorí študenti robili s LLM a čo im fungovalo?") def test_advisor_question_keeps_the_named_profile_a_student(): plan = build_question_plan("Kto vedie Jána Ptáka?", CATALOG, classify=fake(type="advisor_of_student", people=["Jána Ptáka"])) assert plan.spec.intent == "advisors" and [p.role for p in plan.spec.people] == ["student"] def test_negation_without_a_topic_is_not_routed_as_exclusion(): decision = C.CatalogDecision(type="who_did_topic", topic="Python", exclude="Python") assert C.route_for(decision, {"matches": []}) is None def test_literal_safety_net_and_other_activity_bucket(monkeypatch): layer, index, by_key = layer_and_index(monkeypatch) extra = unit("u10", "Eva Nová", "result", "Je urobené vyhodnotenie WER na korpuse.") index["units"].append(extra) graph = A.ActivityGraph(layer, index) # "WER" was never extracted as a tool; a performed record naming it still counts. gid = by_key["whisper"] graph.canonical["wer"] = gid # resolve to some concept so the graph path runs found = A.discover(graph, None, topic="WER", canonical="wer", action="evaluated") assert "u10" in {r["id"] for r in found["rows"]} # Recorded work of another kind is not labelled as a mere plan. found = A.discover(graph, None, topic="RAG", canonical="rag", action="annotated") notes = " ".join(found["notes"]) assert "nie však priamo požadovanú činnosť" in notes and "téma je v zadaní" not in notes