72 lines
4.2 KiB
Python
72 lines
4.2 KiB
Python
"""Gold set: expectations derived from sources and deterministic scoring."""
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools.evaluation import gold_set as G
|
|
|
|
|
|
def wiki(tmp_path):
|
|
root = tmp_path / "wiki"
|
|
for key, body in {
|
|
"2022/eva_nova": "---\ntitle: Eva Nová\ntaxonomy:\n category: [bp2026]\n tag: [rag,nlp]\n author: Daniel Hladek\n---\n"
|
|
"rok začiatku štúdia: 2022\n# Bakalárska práca 2026\nkonzultant Ján Kováč\nStretnutie 5.11.2026\n- Funguje chatbot.\n",
|
|
"2022/adam_novy": "---\ntitle: Adam Nový\ntaxonomy:\n category: [dp2027]\n tag: [nlp]\n author: Maroš Harahus\n---\n"
|
|
"# Bakalárska práca 2025\n# Diplomová práca 2027\nStretnutie 1.10.2026\n",
|
|
}.items():
|
|
path = root / "pages" / "students" / key / "README.md"
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(body, encoding="utf-8")
|
|
return root
|
|
|
|
|
|
def test_expectations_are_derived_from_the_profiles(tmp_path):
|
|
facts = G.profiles(wiki(tmp_path))
|
|
expect, used = G.derive({"id": "x", "derive": {"kind": "supervisor", "profile": "2022/adam_novy"}}, facts)
|
|
assert expect["text_all"] == ["Maroš Harahus"] and used == ["2022/adam_novy"]
|
|
assert G.derive({"id": "x", "derive": {"kind": "consultant", "profile": "2022/eva_nova"}}, facts)[0]["text_all"] == ["Ján Kováč"]
|
|
assert G.derive({"id": "x", "derive": {"kind": "count_of", "supervisor": "Daniel Hladek"}}, facts)[0]["number"] == 1
|
|
both = G.derive({"id": "x", "derive": {"kind": "both_types", "types": ["bachelor_thesis", "diploma_thesis"]}}, facts)[0]
|
|
assert both["people_all"] == ["Adam Nový"] and both["number"] == 1
|
|
assert "2026-11-05" in G.derive({"id": "x", "derive": {"kind": "latest_meeting", "profile": "2022/eva_nova"}}, facts)[0]["text_any"]
|
|
assert G.derive({"id": "x", "derive": {"kind": "tag", "tag": "rag"}}, facts)[0]["people_allowed"] == ["Eva Nová"]
|
|
|
|
|
|
def test_unverifiable_expectations_are_rejected(tmp_path):
|
|
facts = G.profiles(wiki(tmp_path))
|
|
with pytest.raises(ValueError): # evidence pattern not in the student's own profile
|
|
G.derive({"id": "x", "expect": {"people_all": ["Adam Nový"]}, "evidence": {"Adam Nový": "chatbot"}}, facts)
|
|
with pytest.raises(ValueError): # unknown person
|
|
G.derive({"id": "x", "expect": {"people_all": ["Nikto Neznámy"]}}, facts)
|
|
with pytest.raises(ValueError): # a literal phrase that the profile does not contain
|
|
G.derive({"id": "x", "derive": {"kind": "literal", "phrases": {"2022/eva_nova": "Ukončil predčasne"}}}, facts)
|
|
|
|
|
|
PEOPLE = ["Eva Nová", "Adam Nový", "Ján Holp"]
|
|
|
|
|
|
def test_scoring_people_lists_refusals_and_numbers():
|
|
case = {"expect": {"people_all": ["Eva Nová"], "people_none": ["Ján Holp"]}}
|
|
good = {"status": "ok", "answer": "**Eva Nová**\n\n- text\n\nKonzultovať s Jánom Holpom."}
|
|
assert G.score(case, good, PEOPLE)["pass"] # a name inside prose is not a listed person
|
|
bad = {"status": "ok", "answer": "**Eva Nová**\n\n**Ján Holp**\n\n- text"}
|
|
result = G.score(case, bad, PEOPLE)
|
|
assert not result["pass"] and result["precision"] == 0.5
|
|
refusal = {"expect": {"status": "refusal"}}
|
|
assert G.score(refusal, {"status": "ambiguous", "reason": "request_out_of_scope", "answer": "x"}, PEOPLE)["pass"]
|
|
assert not G.score(refusal, {"status": "ambiguous", "reason": "missing_subject", "answer": "x"}, PEOPLE)["pass"]
|
|
assert G.score({"expect": {"number": 69}}, {"status": "ok", "answer": "počet: 69."}, PEOPLE)["pass"]
|
|
assert not G.score({"expect": {"number": 69}}, {"status": "ok", "answer": "počet: 169."}, PEOPLE)["pass"]
|
|
allowed = {"expect": {"people_allowed": ["Eva Nová"], "min_people": 1}}
|
|
assert not G.score(allowed, {"status": "ok", "answer": "Eva Nová má tag.\nAdam Nový má tag."}, PEOPLE)["pass"]
|
|
|
|
|
|
def test_frozen_set_is_complete_and_consistent():
|
|
frozen = json.loads(G.FROZEN.read_text(encoding="utf-8"))
|
|
assert len(frozen["cases"]) >= 100
|
|
assert len({c["id"] for c in frozen["cases"]}) == len(frozen["cases"])
|
|
assert {c["engine"] for c in frozen["cases"]} == {"rag", "graphrag"}
|
|
for case in frozen["cases"]:
|
|
assert case["expect"], case["id"]
|
|
assert all(name in frozen["people"] for name in case["expect"].get("people_all", []))
|