112 lines
4.6 KiB
Python
112 lines
4.6 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from tools.graphrag.evaluation.source_oracle import load_source_oracle, supporting_documents
|
|
from tools.graphrag.evaluation.benchmark_v2 import make_case, validate_case
|
|
|
|
|
|
def source_fixture(tmp_path: Path):
|
|
root = tmp_path / 'wiki'
|
|
profile = 'pages/students/2020/eva_nova/README.md'
|
|
path = root / profile
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text('''---
|
|
title: Eva Nová
|
|
taxonomy:
|
|
category: [bp2025]
|
|
tag: [rag]
|
|
author: Autor Správny
|
|
---
|
|
Rok začiatku štúdia: 2020
|
|
# Diplomový projekt 2024
|
|
Téma: Výskum jazyka pomocou modelov
|
|
## Bakalárska práca 2025
|
|
Názov: Overovanie jazykových modelov
|
|
''')
|
|
child = root / 'pages/students/2020/eva_nova/projekt/README.md'
|
|
child.parent.mkdir(parents=True)
|
|
child.write_text('''---
|
|
title: Pomocný projekt
|
|
taxonomy:
|
|
category: [tp2023]
|
|
tag: [python]
|
|
author: Autor Iný
|
|
---
|
|
Pomocný dokument projektu.
|
|
''')
|
|
return load_source_oracle(root), profile
|
|
|
|
|
|
def test_raw_oracle_distinguishes_year_roles_and_academic_types(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
person = oracle['people']['Eva Nová']
|
|
assert person['start_year'] == 2020
|
|
assert {(row['academic_kind'], row['academic_type'], row['year']) for row in person['academics']} == {
|
|
('Work', 'diploma_project', 2024), ('Thesis', 'bachelor_thesis', 2025)}
|
|
for row in person['academics']:
|
|
lines = oracle['documents'][row['document_path']]['raw_text'].splitlines()
|
|
assert row['title'] in lines[row['title_line'] - 1]
|
|
assert str(row['year']) in lines[row['heading_line'] - 1]
|
|
|
|
|
|
def test_categories_include_child_documents_without_becoming_work_keywords(tmp_path):
|
|
oracle, profile = source_fixture(tmp_path)
|
|
assert oracle['people']['Eva Nová']['categories'] == ['bp2025', 'tp2023']
|
|
assert set(oracle['people']['Eva Nová']['tags']) == {'rag', 'python'}
|
|
assert not supporting_documents(oracle, 'Eva Nová', keyword='jazykové modely')
|
|
assert supporting_documents(oracle, 'Eva Nová', category='bp2025', document_author='Autor Správny') == [profile]
|
|
assert not supporting_documents(oracle, 'Eva Nová', category='bp2025', document_author='Autor Iný')
|
|
|
|
|
|
def test_source_audit_rejects_falsely_negative_existing_tag(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
case = make_case('example', 'dev', 'negative_lookup', 'Má Eva Nová kľúčové slovo rag?',
|
|
['Eva Nová'], oracle, [], no_answer=True,
|
|
negative_constraints={'keyword': 'rag'})
|
|
assert 'negative_conjunction_actually_satisfied' in validate_case(case, oracle)
|
|
case['graph']['negative_constraints']['keyword'] = 'neexistujucitag'
|
|
assert validate_case(case, oracle) == []
|
|
|
|
|
|
def test_source_audit_rejects_wrong_count(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
case = make_case('count', 'dev', 'candidate_set_aggregation',
|
|
'Z kandidátov Eva Nová vyber osoby s prácou v roku 2025. Koľko ich je?',
|
|
['Eva Nová'], oracle, [], expected_set=[], set_field='person', count=0)
|
|
assert 'incorrect_candidate_set_or_count' in validate_case(case, oracle)
|
|
|
|
|
|
def test_source_audit_rejects_faked_partition_profiles(tmp_path):
|
|
oracle, _ = source_fixture(tmp_path)
|
|
case = make_case('example', 'dev', 'person_title_year', 'Aká práca?', ['Eva Nová'], oracle,
|
|
[{'person': 'Eva Nová', 'year': 2025}])
|
|
case['graph']['source_profiles'] = ['pages/students/fake/README.md']
|
|
assert 'incorrect_source_profile_partition' in validate_case(case, oracle)
|
|
|
|
|
|
def test_academic_range_uses_supported_category_end_year(tmp_path):
|
|
oracle, profile = source_fixture(tmp_path)
|
|
path = Path(oracle['wiki_root']) / profile
|
|
path.write_text(path.read_text().replace('bp2025', 'bp2026').replace(
|
|
'Bakalárska práca 2025', 'Bakalárska práca 2025/26'))
|
|
person = load_source_oracle(Path(oracle['wiki_root']))['people']['Eva Nová']
|
|
thesis = next(row for row in person['academics'] if row['academic_kind'] == 'Thesis')
|
|
assert thesis['year'] == 2026
|
|
|
|
|
|
def test_bachelor_project_title_does_not_become_next_year_thesis(tmp_path):
|
|
root = tmp_path / 'wiki'
|
|
path = root / 'pages/students/2020/peter/README.md'
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text('''---
|
|
title: Peter Vzor
|
|
taxonomy:
|
|
category: [bp2025]
|
|
---
|
|
## Bakalársky projekt 2024
|
|
Názov: Overovanie automatických odpovedí
|
|
''')
|
|
records = load_source_oracle(root)['people']['Peter Vzor']['academics']
|
|
assert [(r['academic_kind'], r['academic_type'], r['year']) for r in records] == [('Work', 'bachelor_project', 2024)]
|