from __future__ import annotations from pathlib import Path from tools.graphrag.evaluation.source_oracle import load_source_oracle, supporting_documents from tools.graphrag.evaluation.benchmark_v2 import make_case, validate_case def source_fixture(tmp_path: Path): root = tmp_path / "wiki" profile = "pages/students/2020/eva_nova/README.md" path = root / profile path.parent.mkdir(parents=True) path.write_text("""--- title: Eva Nová taxonomy: category: [bp2025] tag: [rag] author: Autor Správny --- Rok začiatku štúdia: 2020 # Diplomový projekt 2024 Téma: Výskum jazyka pomocou modelov ## Bakalárska práca 2025 Názov: Overovanie jazykových modelov """) child = root / "pages/students/2020/eva_nova/projekt/README.md" child.parent.mkdir(parents=True) child.write_text("""--- title: Pomocný projekt taxonomy: category: [tp2023] tag: [python] author: Autor Iný --- Pomocný dokument projektu. """) return load_source_oracle(root), profile def test_raw_oracle_distinguishes_year_roles_and_academic_types(tmp_path): oracle, _ = source_fixture(tmp_path) person = oracle["people"]["Eva Nová"] assert person["start_year"] == 2020 assert { (row["academic_kind"], row["academic_type"], row["year"]) for row in person["academics"] } == {("Work", "diploma_project", 2024), ("Thesis", "bachelor_thesis", 2025)} for row in person["academics"]: lines = oracle["documents"][row["document_path"]]["raw_text"].splitlines() assert row["title"] in lines[row["title_line"] - 1] assert str(row["year"]) in lines[row["heading_line"] - 1] def test_categories_include_child_documents_without_becoming_work_keywords(tmp_path): oracle, profile = source_fixture(tmp_path) assert oracle["people"]["Eva Nová"]["categories"] == ["bp2025", "tp2023"] assert set(oracle["people"]["Eva Nová"]["tags"]) == {"rag", "python"} assert not supporting_documents(oracle, "Eva Nová", keyword="jazykové modely") assert supporting_documents( oracle, "Eva Nová", category="bp2025", document_author="Autor Správny" ) == [profile] assert not supporting_documents( oracle, "Eva Nová", category="bp2025", document_author="Autor Iný" ) def test_source_audit_rejects_falsely_negative_existing_tag(tmp_path): oracle, _ = source_fixture(tmp_path) case = make_case( "example", "dev", "negative_lookup", "Má Eva Nová kľúčové slovo rag?", ["Eva Nová"], oracle, [], no_answer=True, negative_constraints={"keyword": "rag"}, ) assert "negative_conjunction_actually_satisfied" in validate_case(case, oracle) case["graph"]["negative_constraints"]["keyword"] = "neexistujucitag" assert validate_case(case, oracle) == [] def test_source_audit_rejects_wrong_count(tmp_path): oracle, _ = source_fixture(tmp_path) case = make_case( "count", "dev", "candidate_set_aggregation", "Z kandidátov Eva Nová vyber osoby s prácou v roku 2025. Koľko ich je?", ["Eva Nová"], oracle, [], expected_set=[], set_field="person", count=0, ) assert "incorrect_candidate_set_or_count" in validate_case(case, oracle) def test_source_audit_rejects_faked_partition_profiles(tmp_path): oracle, _ = source_fixture(tmp_path) case = make_case( "example", "dev", "person_title_year", "Aká práca?", ["Eva Nová"], oracle, [{"person": "Eva Nová", "year": 2025}], ) case["graph"]["source_profiles"] = ["pages/students/fake/README.md"] assert "incorrect_source_profile_partition" in validate_case(case, oracle) def test_academic_range_uses_supported_category_end_year(tmp_path): oracle, profile = source_fixture(tmp_path) path = Path(oracle["wiki_root"]) / profile path.write_text( path.read_text() .replace("bp2025", "bp2026") .replace("Bakalárska práca 2025", "Bakalárska práca 2025/26") ) person = load_source_oracle(Path(oracle["wiki_root"]))["people"]["Eva Nová"] thesis = next(row for row in person["academics"] if row["academic_kind"] == "Thesis") assert thesis["year"] == 2026 def test_bachelor_project_title_does_not_become_next_year_thesis(tmp_path): root = tmp_path / "wiki" path = root / "pages/students/2020/peter/README.md" path.parent.mkdir(parents=True) path.write_text("""--- title: Peter Vzor taxonomy: category: [bp2025] --- ## Bakalársky projekt 2024 Názov: Overovanie automatických odpovedí """) records = load_source_oracle(root)["people"]["Peter Vzor"]["academics"] assert [(r["academic_kind"], r["academic_type"], r["year"]) for r in records] == [ ("Work", "bachelor_project", 2024) ]