146 lines
4.8 KiB
Python
146 lines
4.8 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from tools.graphrag.evaluation.source_oracle import load_source_oracle, supporting_documents
|
|
from tools.graphrag.evaluation.benchmark_v2 import make_case, validate_case
|
|
|
|
|
|
def source_fixture(tmp_path: Path):
|
|
root = tmp_path / "wiki"
|
|
profile = "pages/students/2020/eva_nova/README.md"
|
|
path = root / profile
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text("""---
|
|
title: Eva Nová
|
|
taxonomy:
|
|
category: [bp2025]
|
|
tag: [rag]
|
|
author: Autor Správny
|
|
---
|
|
Rok začiatku štúdia: 2020
|
|
# Diplomový projekt 2024
|
|
Téma: Výskum jazyka pomocou modelov
|
|
## Bakalárska práca 2025
|
|
Názov: Overovanie jazykových modelov
|
|
""")
|
|
child = root / "pages/students/2020/eva_nova/projekt/README.md"
|
|
child.parent.mkdir(parents=True)
|
|
child.write_text("""---
|
|
title: Pomocný projekt
|
|
taxonomy:
|
|
category: [tp2023]
|
|
tag: [python]
|
|
author: Autor Iný
|
|
---
|
|
Pomocný dokument projektu.
|
|
""")
|
|
return load_source_oracle(root), profile
|
|
|
|
|
|
def test_raw_oracle_distinguishes_year_roles_and_academic_types(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
person = oracle["people"]["Eva Nová"]
|
|
assert person["start_year"] == 2020
|
|
assert {
|
|
(row["academic_kind"], row["academic_type"], row["year"]) for row in person["academics"]
|
|
} == {("Work", "diploma_project", 2024), ("Thesis", "bachelor_thesis", 2025)}
|
|
for row in person["academics"]:
|
|
lines = oracle["documents"][row["document_path"]]["raw_text"].splitlines()
|
|
assert row["title"] in lines[row["title_line"] - 1]
|
|
assert str(row["year"]) in lines[row["heading_line"] - 1]
|
|
|
|
|
|
def test_categories_include_child_documents_without_becoming_work_keywords(tmp_path):
|
|
oracle, profile = source_fixture(tmp_path)
|
|
assert oracle["people"]["Eva Nová"]["categories"] == ["bp2025", "tp2023"]
|
|
assert set(oracle["people"]["Eva Nová"]["tags"]) == {"rag", "python"}
|
|
assert not supporting_documents(oracle, "Eva Nová", keyword="jazykové modely")
|
|
assert supporting_documents(
|
|
oracle, "Eva Nová", category="bp2025", document_author="Autor Správny"
|
|
) == [profile]
|
|
assert not supporting_documents(
|
|
oracle, "Eva Nová", category="bp2025", document_author="Autor Iný"
|
|
)
|
|
|
|
|
|
def test_source_audit_rejects_falsely_negative_existing_tag(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
case = make_case(
|
|
"example",
|
|
"dev",
|
|
"negative_lookup",
|
|
"Má Eva Nová kľúčové slovo rag?",
|
|
["Eva Nová"],
|
|
oracle,
|
|
[],
|
|
no_answer=True,
|
|
negative_constraints={"keyword": "rag"},
|
|
)
|
|
assert "negative_conjunction_actually_satisfied" in validate_case(case, oracle)
|
|
case["graph"]["negative_constraints"]["keyword"] = "neexistujucitag"
|
|
assert validate_case(case, oracle) == []
|
|
|
|
|
|
def test_source_audit_rejects_wrong_count(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
case = make_case(
|
|
"count",
|
|
"dev",
|
|
"candidate_set_aggregation",
|
|
"Z kandidátov Eva Nová vyber osoby s prácou v roku 2025. Koľko ich je?",
|
|
["Eva Nová"],
|
|
oracle,
|
|
[],
|
|
expected_set=[],
|
|
set_field="person",
|
|
count=0,
|
|
)
|
|
assert "incorrect_candidate_set_or_count" in validate_case(case, oracle)
|
|
|
|
|
|
def test_source_audit_rejects_faked_partition_profiles(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
case = make_case(
|
|
"example",
|
|
"dev",
|
|
"person_title_year",
|
|
"Aká práca?",
|
|
["Eva Nová"],
|
|
oracle,
|
|
[{"person": "Eva Nová", "year": 2025}],
|
|
)
|
|
case["graph"]["source_profiles"] = ["pages/students/fake/README.md"]
|
|
assert "incorrect_source_profile_partition" in validate_case(case, oracle)
|
|
|
|
|
|
def test_academic_range_uses_supported_category_end_year(tmp_path):
|
|
oracle, profile = source_fixture(tmp_path)
|
|
path = Path(oracle["wiki_root"]) / profile
|
|
path.write_text(
|
|
path.read_text()
|
|
.replace("bp2025", "bp2026")
|
|
.replace("Bakalárska práca 2025", "Bakalárska práca 2025/26")
|
|
)
|
|
person = load_source_oracle(Path(oracle["wiki_root"]))["people"]["Eva Nová"]
|
|
thesis = next(row for row in person["academics"] if row["academic_kind"] == "Thesis")
|
|
assert thesis["year"] == 2026
|
|
|
|
|
|
def test_bachelor_project_title_does_not_become_next_year_thesis(tmp_path):
|
|
root = tmp_path / "wiki"
|
|
path = root / "pages/students/2020/peter/README.md"
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text("""---
|
|
title: Peter Vzor
|
|
taxonomy:
|
|
category: [bp2025]
|
|
---
|
|
## Bakalársky projekt 2024
|
|
Názov: Overovanie automatických odpovedí
|
|
""")
|
|
records = load_source_oracle(root)["people"]["Peter Vzor"]["academics"]
|
|
assert [(r["academic_kind"], r["academic_type"], r["year"]) for r in records] == [
|
|
("Work", "bachelor_project", 2024)
|
|
]
|