185 lines
11 KiB
Python
185 lines
11 KiB
Python
"""Activity layer (who did what with which tool) and the question catalog."""
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.knowledge import activities as A
|
|
from tools.answers import question_catalog as C
|
|
from tools.answers.question_plan import build_question_plan, direct_spec
|
|
|
|
|
|
def unit(uid, person, phase, text, *, kind=None, work="Bakalárska práca 2021", work_id=None, date=None):
|
|
path = f"pages/students/2020/{person.split()[-1].lower()}/README.md"
|
|
return {"id": uid, "person": person, "source_path": path, "source_url": "https://zp/" + path,
|
|
"line": int(uid[1:]) if uid[1:].isdigit() else 1, "work_label": work, "work_type": "bachelor_thesis",
|
|
"work_year": 2021, "work_id": work_id or (person + work), "kind": kind or {"result": "result",
|
|
"progress": "progress", "planned": "plan", "assignment": "assignment"}.get(phase, "context"),
|
|
"phase": phase, "text": text, "section_scope": "work", "actor": person,
|
|
"actor_status": "profile_context", "annotation_status": "knowledge", "record_date": date}
|
|
|
|
|
|
UNITS = [
|
|
unit("u1", "Andrii Pervashov", "result", "Na korpuse funguje rozpoznávanie reči pomocou Whisper."),
|
|
unit("u2", "Eva Nová", "progress", "Rozpoznávanie nenávistnej reči pomocou transformers."),
|
|
unit("u3", "Samuel Horáni", "unknown", "Chatbot pre mesto", kind="topic"),
|
|
unit("u4", "Samuel Horáni", "result", "Vytvorené prepojenie s databázou a RASA."),
|
|
unit("u5", "Ján Pták", "result", "Prečítané články o chatbotoch."),
|
|
unit("u6", "Peter Plán", "planned", "Vytvorte chatbota pre študentov."),
|
|
unit("u7", "Oleh Poiasnik", "result", "Implementovaný RAG systém s LangChain."),
|
|
unit("u8", "Ján Malinovský", "result", "Funguje retrieval augmented generation nad grafom."),
|
|
unit("u9", "Andrii Pervashov", "result", "Bolo urobené dotrénovanie modelu Whisper."),
|
|
]
|
|
RECORDS = [
|
|
{"unit": "u1", "action": "used", "outcome": "works", "topic": {"text": "rozpoznávanie reči", "canonical": "speech recognition"},
|
|
"tools": [{"text": "Whisper", "canonical": "whisper", "type": "ml_model"}]},
|
|
{"unit": "u2", "action": "implemented", "outcome": "unknown", "topic": {"text": "Rozpoznávanie nenávistnej reči",
|
|
"canonical": "hate speech detection"}, "tools": []},
|
|
{"unit": "u3", "action": "implemented", "outcome": "unknown", "topic": {"text": "Chatbot", "canonical": "chatbot"}, "tools": []},
|
|
{"unit": "u4", "action": "implemented", "outcome": "works", "topic": {"text": "prepojenie s databázou",
|
|
"canonical": "database integration"}, "tools": [{"text": "RASA", "canonical": "rasa", "type": "framework"}]},
|
|
{"unit": "u5", "action": "studied", "outcome": "unknown", "topic": {"text": "chatbotoch", "canonical": "chatbots"}, "tools": []},
|
|
{"unit": "u6", "action": "implemented", "outcome": "unknown", "topic": {"text": "chatbota", "canonical": "chatbot"}, "tools": []},
|
|
{"unit": "u7", "action": "implemented", "outcome": "works", "topic": {"text": "RAG", "canonical": "rag"},
|
|
"tools": [{"text": "LangChain", "canonical": "langchain", "type": "library"}]},
|
|
{"unit": "u8", "action": "used", "outcome": "works", "topic": {"text": "retrieval augmented generation",
|
|
"canonical": "retrieval augmented generation"}, "tools": []},
|
|
{"unit": "u9", "action": "fine_tuned", "outcome": "works", "topic": None,
|
|
"tools": [{"text": "Whisper", "canonical": "whisper", "type": "ml_model"}]},
|
|
]
|
|
|
|
|
|
def layer_and_index(monkeypatch):
|
|
import numpy as np
|
|
monkeypatch.setattr(A, "_embed", lambda texts: np.zeros((len(texts), 4), dtype="float32"))
|
|
concepts, by_key = A.unify(RECORDS, None)
|
|
activities = [{"unit": r["unit"], "action": r["action"], "outcome": r["outcome"],
|
|
"topic": by_key.get(r["topic"]["canonical"]) if r["topic"] else None,
|
|
"topic_text": r["topic"]["text"] if r["topic"] else None,
|
|
"tools": [{"concept": by_key[t["canonical"]], "text": t["text"]} for t in r["tools"]]} for r in RECORDS]
|
|
index = {"version": "k1", "index_version": {"version": "v1"}, "units": UNITS}
|
|
ids = sorted(concepts)
|
|
layer = {"schema": A.ACTIVITY_SCHEMA, "extract_version": A.EXTRACT_VERSION, "knowledge_version": "k1",
|
|
"index_version": {"version": "v1"}, "built_at": "now", "concepts": concepts, "activities": activities,
|
|
"concept_vectors": {"ids": [], "shape": [0, 0], "data": __import__("base64").b64encode(
|
|
__import__("zlib").compress(b"")).decode()}}
|
|
return layer, index, by_key
|
|
|
|
|
|
def test_extraction_keeps_only_literal_phrases():
|
|
item = A._lenient_item({"id": "U1", "activities": [
|
|
{"action": "trained", "topic": {"text": "rozpoznávanie reči", "canonical": "Speech Recognition"},
|
|
"tools": [{"text": "Whisper", "canonical": "whisper", "type": "speech"}, {"text": "GPT-9", "canonical": "gpt"}]},
|
|
{"action": "invented", "topic": {"text": "niečo iné", "canonical": "x"}}]})
|
|
acts = A.validated_activities({"text": "Bolo natrénované Rozpoznávanie reči s Whisper."}, item)
|
|
assert acts[0]["topic"] == {"text": "rozpoznávanie reči", "canonical": "speech recognition"}
|
|
assert [t["text"] for t in acts[0]["tools"]] == ["Whisper"] and acts[0]["tools"][0]["type"] == "other"
|
|
assert len(acts) == 1 # the second activity names no literal phrase of the unit
|
|
|
|
|
|
def test_unification_merges_spelling_and_acronyms_not_neighbours(monkeypatch):
|
|
layer, index, by_key = layer_and_index(monkeypatch)
|
|
assert by_key["chatbot"] == by_key["chatbot"] and by_key["rag"] == by_key["retrieval augmented generation"]
|
|
assert by_key["speech recognition"] != by_key["hate speech detection"]
|
|
assert A.canonical_key("Chatbots") == "chatbot"
|
|
|
|
|
|
def test_speech_recognition_does_not_include_hate_speech(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="rozpoznávaniu reči", canonical="speech recognition")
|
|
people = {r["person"] for r in found["rows"]}
|
|
assert people == {"Andrii Pervashov"}
|
|
|
|
|
|
def test_thesis_topic_counts_its_recorded_work_and_reading_is_not_doing(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="chatbota", canonical="chatbot")
|
|
rows = {r["id"] for r in found["rows"]}
|
|
assert {"u3", "u4"} <= rows # Horáni: thesis about a chatbot + its RASA result
|
|
notes = " ".join(found["notes"])
|
|
assert "Samuel Horáni — téma práce" in notes
|
|
assert "Peter Plán — téma je v zadaní" in notes
|
|
assert "Ján Pták — záznamy dokladajú len štúdium" in notes
|
|
order = [r["person"] for r in found["rows"]]
|
|
assert order.index("Samuel Horáni") < order.index("Peter Plán") < order.index("Ján Pták")
|
|
|
|
|
|
def test_rag_acronym_and_long_name_are_one_topic_and_exclusion_applies(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="RAG", canonical="retrieval augmented generation")
|
|
assert {r["person"] for r in found["rows"]} == {"Oleh Poiasnik", "Ján Malinovský"}
|
|
found = A.discover(graph, None, topic="RAG", canonical="rag", excluded=("LangChain",))
|
|
assert {r["person"] for r in found["rows"]} == {"Ján Malinovský"} and found["excluded"] == ["Oleh Poiasnik"]
|
|
|
|
|
|
def test_action_filters_and_generic_topic(monkeypatch):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
graph = A.ActivityGraph(layer, index)
|
|
found = A.discover(graph, None, topic="model", action="fine_tuned")
|
|
assert {r["person"] for r in found["rows"]} == {"Andrii Pervashov"}
|
|
assert A.question_action("Kto dotrénoval Whisper?") == "fine_tuned"
|
|
assert A.question_action("Kto robil chatbota?") == "any"
|
|
assert A.discover(graph, None, topic="kvantová chémia") is None
|
|
|
|
|
|
def test_stale_layer_is_never_served(monkeypatch, tmp_path):
|
|
layer, index, _ = layer_and_index(monkeypatch)
|
|
(tmp_path / "knowledge").mkdir()
|
|
(tmp_path / A.LAYER_FILE).write_text(json.dumps(layer), encoding="utf-8")
|
|
assert A.load_graph(tmp_path, index) is not None
|
|
assert A.load_graph(tmp_path, {**index, "version": "k2"}) is None
|
|
|
|
|
|
def test_catalog_parameters_must_be_literal():
|
|
decision = C.checked(C.CatalogDecision(type="who_did_topic", people=["Eva"], topic="chatbota",
|
|
topic_canonical="Chat-Bot!", exclude="Whisper"), "Kto robil chatbota bez RASA?")
|
|
assert decision.people == [] and decision.topic == "chatbota" and decision.exclude is None
|
|
assert decision.topic_canonical == "chat bot"
|
|
|
|
|
|
CATALOG = [{"kind": "Person", "name": "Maroš Harahus", "source_document": "pages/students/2016/maros_harahus/README.md"},
|
|
{"kind": "Person", "name": "Ján Pták", "source_document": "pages/students/2022/jan_ptak/README.md"}]
|
|
|
|
|
|
def fake(**kwargs):
|
|
return lambda question: C.CatalogDecision(**kwargs)
|
|
|
|
|
|
def test_catalog_route_overrides_rules_and_keeps_literals():
|
|
plan = build_question_plan("Ktorých študentov vedie Maroš Harahus?", CATALOG,
|
|
classify=fake(type="students_of_advisor", people=["Maroš Harahus"]))
|
|
assert plan.spec.intent == "supervisor" and plan.spec.people[0].role == "supervisor"
|
|
plan = build_question_plan("Kto robil chatbota, ale nepoužil RASA?", CATALOG,
|
|
classify=fake(type="who_did_topic", topic="chatbota", topic_canonical="chatbot", exclude="RASA"))
|
|
assert plan.spec.intent == "discovery" and plan.spec.topic == "chatbota"
|
|
assert [(c.kind, c.quote) for c in plan.spec.conditions] == [("exclusion", "rasa")]
|
|
plan = build_question_plan("Čo robil Ján Pták v októbri 2026?", CATALOG,
|
|
classify=fake(type="activity_in_period", people=["Ján Pták"]))
|
|
assert plan.spec.intent == "progress" and plan.spec.period.start == "2026-10-01"
|
|
|
|
|
|
def test_catalog_failure_or_invalid_route_keeps_rule_plan():
|
|
rules = build_question_plan("Kto je vedúci práce Ján Pták?", CATALOG)
|
|
for classify in (lambda q: None, fake(type="unclear"), fake(type="compare_students", people=["Ján Pták"])):
|
|
assert build_question_plan("Kto je vedúci práce Ján Pták?", CATALOG, classify=classify).spec == rules.spec
|
|
|
|
|
|
def test_out_of_scope_refused_only_without_wiki_subject():
|
|
off = build_question_plan("Aké bude zajtra počasie v Košiciach?", CATALOG, classify=fake(type="out_of_scope"))
|
|
assert off.reason == "request_out_of_scope"
|
|
wiki = build_question_plan("Ktoré témy riešili študenti?", CATALOG, classify=fake(type="out_of_scope"))
|
|
assert wiki.reason != "request_out_of_scope"
|
|
|
|
|
|
def test_runtime_prompt_contains_every_type_and_demos():
|
|
program = {"instructions": C.INSTRUCTIONS, "demos": [{"question": "Kto robil chatbota?",
|
|
"decision": {"type": "who_did_topic", "topic": "chatbota"}}]}
|
|
messages = C.messages_for("Kto robil RAG?", program)
|
|
assert all(t in messages[0]["content"] for t in C.TYPE_DESCRIPTIONS)
|
|
assert [m["role"] for m in messages] == ["system", "user", "assistant", "user"]
|
|
reply = {"choices": [{"message": {"content": json.dumps({"type": "who_did_topic", "topic": "RAG"})}}]}
|
|
C._cache.clear()
|
|
assert C.classify("Kto robil RAG?", lambda m: reply, program=program).topic == "RAG"
|