86 lines
3.1 KiB
Python
86 lines
3.1 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from tools.graphrag.evaluation.freeze_graphrag import make_freeze
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file, verify_freeze
|
|
|
|
|
|
@pytest.fixture
|
|
def freeze_case():
|
|
questions = [{"id": "development-a", "split": "dev"}, {"id": "holdout-a", "split": "test"}]
|
|
fingerprints = {
|
|
"implementation": {"search.py": "version-one"},
|
|
"dataset_sha256": "dataset",
|
|
"inputs": {},
|
|
}
|
|
graph = {"sha256": "graph", "nodes": 1, "relationships": 0}
|
|
audit = {
|
|
"questions": [{"id": q["id"]} for q in questions],
|
|
"by_split": {},
|
|
"dataset_sha256": "dataset",
|
|
"inputs": {},
|
|
}
|
|
dev = {
|
|
"configuration": {"split": "dev", "limit": 40, "max_entities": 8},
|
|
"summary": {"error_count": 0},
|
|
"questions": [{"id": "development-a", "error": None}],
|
|
"run_metadata": {"fingerprints": fingerprints, "graph": graph},
|
|
}
|
|
return questions, fingerprints, graph, audit, dev
|
|
|
|
|
|
def test_freeze_rejects_partial_dev(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
dev["questions"] = []
|
|
with pytest.raises(RuntimeError, match="complete DEV"):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
def test_freeze_rejects_runtime_errors(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
dev["questions"][0]["error"] = "database unavailable"
|
|
with pytest.raises(RuntimeError, match="runtime errors"):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
def test_freeze_rejects_code_changed_after_dev(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
changed = copy.deepcopy(fingerprints)
|
|
changed["implementation"]["search.py"] = "version-two"
|
|
with pytest.raises(RuntimeError, match="changed after DEV"):
|
|
make_freeze(dev, questions, changed, audit)
|
|
|
|
|
|
def test_freeze_rejects_stale_source_audit(freeze_case):
|
|
questions, fingerprints, _, audit, dev = freeze_case
|
|
audit["inputs"] = {"documents": "old-snapshot"}
|
|
with pytest.raises(RuntimeError, match="audit does not match"):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["code", "graph", "parameters", "audit"])
|
|
def test_holdout_rejects_changes_after_freeze(freeze_case, tmp_path, mutation):
|
|
questions, fingerprints, graph, audit, dev = freeze_case
|
|
manifest = make_freeze(dev, questions, fingerprints, audit)
|
|
audit_path = tmp_path / "audit.json"
|
|
audit_path.write_text("original-audit")
|
|
manifest["audit_sha256"] = sha256_file(audit_path)
|
|
metadata = {"fingerprints": copy.deepcopy(fingerprints)}
|
|
actual_graph = dict(graph)
|
|
limit = 40
|
|
if mutation == "code":
|
|
metadata["fingerprints"]["implementation"]["search.py"] = "changed"
|
|
elif mutation == "graph":
|
|
actual_graph["sha256"] = "changed"
|
|
elif mutation == "parameters":
|
|
limit = 41
|
|
else:
|
|
audit_path.write_text("changed-audit")
|
|
with pytest.raises(RuntimeError, match="TEST refused"):
|
|
verify_freeze(
|
|
manifest, metadata, actual_graph, limit=limit, max_entities=8, audit=audit_path
|
|
)
|