73 lines
3.0 KiB
Python
73 lines
3.0 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from tools.graphrag.evaluation.freeze_graphrag import make_freeze
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file, verify_freeze
|
|
|
|
|
|
@pytest.fixture
|
|
def freeze_case():
|
|
questions = [{'id': 'development-a', 'split': 'dev'}, {'id': 'holdout-a', 'split': 'test'}]
|
|
fingerprints = {'implementation': {'search.py': 'version-one'}, 'dataset_sha256': 'dataset', 'inputs': {}}
|
|
graph = {'sha256': 'graph', 'nodes': 1, 'relationships': 0}
|
|
audit = {'questions': [{'id': q['id']} for q in questions], 'by_split': {},
|
|
'dataset_sha256': 'dataset', 'inputs': {}}
|
|
dev = {'configuration': {'split': 'dev', 'limit': 40, 'max_entities': 8},
|
|
'summary': {'error_count': 0}, 'questions': [{'id': 'development-a', 'error': None}],
|
|
'run_metadata': {'fingerprints': fingerprints, 'graph': graph}}
|
|
return questions, fingerprints, graph, audit, dev
|
|
|
|
|
|
def test_freeze_rejects_partial_dev(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
dev['questions'] = []
|
|
with pytest.raises(RuntimeError, match='complete DEV'):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
def test_freeze_rejects_runtime_errors(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
dev['questions'][0]['error'] = 'database unavailable'
|
|
with pytest.raises(RuntimeError, match='runtime errors'):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
def test_freeze_rejects_code_changed_after_dev(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
changed = copy.deepcopy(fingerprints)
|
|
changed['implementation']['search.py'] = 'version-two'
|
|
with pytest.raises(RuntimeError, match='changed after DEV'):
|
|
make_freeze(dev, questions, changed, audit)
|
|
|
|
|
|
def test_freeze_rejects_stale_source_audit(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
audit['inputs'] = {'documents': 'old-snapshot'}
|
|
with pytest.raises(RuntimeError, match='audit does not match'):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
@pytest.mark.parametrize('mutation', ['code', 'graph', 'parameters', 'audit'])
|
|
def test_holdout_rejects_changes_after_freeze(freeze_case, tmp_path, mutation):
|
|
questions, fingerprints, graph, audit, dev = freeze_case
|
|
manifest = make_freeze(dev, questions, fingerprints, audit)
|
|
audit_path = tmp_path / 'audit.json'
|
|
audit_path.write_text('original-audit')
|
|
manifest['audit_sha256'] = sha256_file(audit_path)
|
|
metadata = {'fingerprints': copy.deepcopy(fingerprints)}
|
|
actual_graph = dict(graph)
|
|
limit = 40
|
|
if mutation == 'code':
|
|
metadata['fingerprints']['implementation']['search.py'] = 'changed'
|
|
elif mutation == 'graph':
|
|
actual_graph['sha256'] = 'changed'
|
|
elif mutation == 'parameters':
|
|
limit = 41
|
|
else:
|
|
audit_path.write_text('changed-audit')
|
|
with pytest.raises(RuntimeError, match='TEST refused'):
|
|
verify_freeze(manifest, metadata, actual_graph, limit=limit, max_entities=8, audit=audit_path)
|