from __future__ import annotations import copy from pathlib import Path import pytest from tools.graphrag.evaluation.freeze_graphrag import make_freeze from tools.graphrag.evaluation.reproducibility import source_corpus_fingerprint def freeze_inputs(): questions = [ {'id': 'dev-a', 'split': 'dev', 'graph': {'schema_version': 3, 'source_profiles': ['pages/students/a/README.md']}}, {'id': 'test-b', 'split': 'test', 'graph': {'schema_version': 3, 'source_profiles': ['pages/students/b/README.md']}}, ] fingerprints = {'implementation': {}, 'dataset_sha256': 'dataset', 'inputs': {}, 'source_corpus': {'sha256': 'raw-corpus'}} audit = {'dataset_sha256': 'dataset', 'inputs': {}, 'source_corpus': {'sha256': 'raw-corpus'}, 'questions': [{'id': q['id'], 'source_consistent': True} for q in questions], 'by_split': {}} dev = {'configuration': {'split': 'dev', 'limit': 40, 'max_entities': 8}, 'questions': [{'id': 'dev-a', 'error': None}], 'summary': {'error_count': 0}, 'run_metadata': {'fingerprints': fingerprints, 'graph': {'sha256': 'graph'}}} return dev, questions, fingerprints, audit def test_primary_freeze_verifies_disjoint_profiles(): args = freeze_inputs() manifest = make_freeze(*args) assert manifest['partition']['overlap'] == 0 assert manifest['partition']['test_profile_count'] == 1 def test_primary_freeze_rejects_same_profile_across_splits(): dev, questions, fingerprints, audit = freeze_inputs() questions[1]['graph']['source_profiles'] = questions[0]['graph']['source_profiles'] with pytest.raises(RuntimeError, match='profiles overlap'): make_freeze(dev, questions, fingerprints, audit) def test_primary_freeze_rejects_unvalidated_source_expectation(): dev, questions, fingerprints, audit = freeze_inputs() audit['questions'][1]['source_consistent'] = False with pytest.raises(RuntimeError, match='source-inconsistent'): make_freeze(dev, questions, fingerprints, audit) def test_primary_freeze_rejects_stale_raw_markdown_audit(): dev, questions, fingerprints, audit = freeze_inputs() audit['source_corpus'] = {'sha256': 'stale'} with pytest.raises(RuntimeError, match='original Markdown'): make_freeze(dev, questions, fingerprints, audit) def test_raw_source_fingerprint_detects_unindexed_change(tmp_path: Path, monkeypatch): wiki = tmp_path / 'wiki' doc = wiki / 'pages/students/example/README.md' doc.parent.mkdir(parents=True) doc.write_text('---\ntitle: Example\n---\nOriginal source\n') monkeypatch.setenv('ZPWIKI_ROOT', str(wiki)) original = source_corpus_fingerprint() doc.write_text('---\ntitle: Example\n---\nChanged source\n') assert original['sha256'] != source_corpus_fingerprint()['sha256'] def test_freeze_cli_does_not_overwrite_previous_manifest(tmp_path, monkeypatch): from tools.graphrag.evaluation.freeze_graphrag import main previous = tmp_path / 'frozen_v1.json' previous.write_text('historical manifest') monkeypatch.setattr('sys.argv', ['freeze', '--dev-results', 'unused', '--audit', 'unused', '--output', str(previous)]) with pytest.raises(SystemExit): main() assert previous.read_text() == 'historical manifest' def test_dev_overwrite_cannot_replace_archived_results(tmp_path): from argparse import Namespace from tools.graphrag.evaluation.evaluate_graphrag import validate_output_destination directory = tmp_path / 'final_v1' directory.mkdir() previous = directory / 'graphrag_dev.json' previous.write_text('archived DEV') args = Namespace(split='dev', ids='', max_questions=None, overwrite=True, output_dir=directory) with pytest.raises(RuntimeError, match='immutable'): validate_output_destination(args) assert previous.read_text() == 'archived DEV' def test_reproduction_requires_fresh_results_directory(tmp_path): from argparse import Namespace from tools.graphrag.evaluation.evaluate_graphrag import validate_output_destination previous = tmp_path / 'graphrag_test.json' previous.write_text('archived TEST') args = Namespace(split='test', ids='', max_questions=None, overwrite=False, output_dir=tmp_path) with pytest.raises(RuntimeError, match='already exist'): validate_output_destination(args) assert previous.read_text() == 'archived TEST'