from __future__ import annotations from pathlib import Path from tools.graphrag.evaluation.source_oracle import load_source_oracle, supporting_documents from tools.graphrag.evaluation.benchmark_v2 import make_case, validate_case def source_fixture(tmp_path: Path): root = tmp_path / 'wiki' profile = 'pages/students/2020/eva_nova/README.md' path = root / profile path.parent.mkdir(parents=True) path.write_text('''--- title: Eva Nová taxonomy: category: [bp2025] tag: [rag] author: Autor Správny --- Rok začiatku štúdia: 2020 # Diplomový projekt 2024 Téma: Výskum jazyka pomocou modelov ## Bakalárska práca 2025 Názov: Overovanie jazykových modelov ''') child = root / 'pages/students/2020/eva_nova/projekt/README.md' child.parent.mkdir(parents=True) child.write_text('''--- title: Pomocný projekt taxonomy: category: [tp2023] tag: [python] author: Autor Iný --- Pomocný dokument projektu. ''') return load_source_oracle(root), profile def test_raw_oracle_distinguishes_year_roles_and_academic_types(tmp_path): oracle, _ = source_fixture(tmp_path) person = oracle['people']['Eva Nová'] assert person['start_year'] == 2020 assert {(row['academic_kind'], row['academic_type'], row['year']) for row in person['academics']} == { ('Work', 'diploma_project', 2024), ('Thesis', 'bachelor_thesis', 2025)} for row in person['academics']: lines = oracle['documents'][row['document_path']]['raw_text'].splitlines() assert row['title'] in lines[row['title_line'] - 1] assert str(row['year']) in lines[row['heading_line'] - 1] def test_categories_include_child_documents_without_becoming_work_keywords(tmp_path): oracle, profile = source_fixture(tmp_path) assert oracle['people']['Eva Nová']['categories'] == ['bp2025', 'tp2023'] assert set(oracle['people']['Eva Nová']['tags']) == {'rag', 'python'} assert not supporting_documents(oracle, 'Eva Nová', keyword='jazykové modely') assert supporting_documents(oracle, 'Eva Nová', category='bp2025', document_author='Autor Správny') == [profile] assert not supporting_documents(oracle, 'Eva Nová', category='bp2025', document_author='Autor Iný') def test_source_audit_rejects_falsely_negative_existing_tag(tmp_path): oracle, _ = source_fixture(tmp_path) case = make_case('example', 'dev', 'negative_lookup', 'Má Eva Nová kľúčové slovo rag?', ['Eva Nová'], oracle, [], no_answer=True, negative_constraints={'keyword': 'rag'}) assert 'negative_conjunction_actually_satisfied' in validate_case(case, oracle) case['graph']['negative_constraints']['keyword'] = 'neexistujucitag' assert validate_case(case, oracle) == [] def test_source_audit_rejects_wrong_count(tmp_path): oracle, _ = source_fixture(tmp_path) case = make_case('count', 'dev', 'candidate_set_aggregation', 'Z kandidátov Eva Nová vyber osoby s prácou v roku 2025. Koľko ich je?', ['Eva Nová'], oracle, [], expected_set=[], set_field='person', count=0) assert 'incorrect_candidate_set_or_count' in validate_case(case, oracle) def test_source_audit_rejects_faked_partition_profiles(tmp_path): oracle, _ = source_fixture(tmp_path) case = make_case('example', 'dev', 'person_title_year', 'Aká práca?', ['Eva Nová'], oracle, [{'person': 'Eva Nová', 'year': 2025}]) case['graph']['source_profiles'] = ['pages/students/fake/README.md'] assert 'incorrect_source_profile_partition' in validate_case(case, oracle) def test_academic_range_uses_supported_category_end_year(tmp_path): oracle, profile = source_fixture(tmp_path) path = Path(oracle['wiki_root']) / profile path.write_text(path.read_text().replace('bp2025', 'bp2026').replace( 'Bakalárska práca 2025', 'Bakalárska práca 2025/26')) person = load_source_oracle(Path(oracle['wiki_root']))['people']['Eva Nová'] thesis = next(row for row in person['academics'] if row['academic_kind'] == 'Thesis') assert thesis['year'] == 2026