"""Content fingerprints used to freeze retrieval before holdout evaluation.""" from __future__ import annotations import hashlib import json import platform import subprocess from datetime import datetime, timezone from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[3] def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: files = sorted((root / 'tools/graphrag').rglob('*.py')) # requirements.txt and tools/ are present both in the checkout and image. files.append(root / 'requirements.txt') return { 'implementation': {str(path.relative_to(root)): sha256_file(path) for path in files}, 'dataset_sha256': sha256_file(questions), 'inputs': {name: sha256_file(root / name) for name in ['data/documents.json', 'data/chunks.json']}, } def graph_fingerprint(session: Any) -> dict[str, Any]: nodes = session.run( 'MATCH (n) WHERE n.graph_scope = $scope ' 'RETURN labels(n) AS labels, properties(n) AS properties', scope='zpwiki' ).data() edges = session.run( 'MATCH (a)-[r]->(b) WHERE a.graph_scope = $scope AND b.graph_scope = $scope ' 'RETURN labels(a) AS source_labels, coalesce(a.id, a.path, a.name) AS source, ' 'type(r) AS type, properties(r) AS properties, labels(b) AS target_labels, ' 'coalesce(b.id, b.path, b.name) AS target', scope='zpwiki' ).data() # elementId and row order are deliberately excluded: they change on rebuild. canonical = lambda rows: sorted(json.dumps(row, sort_keys=True, ensure_ascii=False, default=str) for row in rows) content = json.dumps({'nodes': canonical(nodes), 'relationships': canonical(edges)}, ensure_ascii=False) return {'sha256': hashlib.sha256(content.encode()).hexdigest(), 'nodes': len(nodes), 'relationships': len(edges)} def run_metadata(questions: Path) -> dict[str, Any]: def revision(path: Path) -> str | None: result = subprocess.run(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True, capture_output=True, check=False) return result.stdout.strip() if result.returncode == 0 else None return {'started_at_utc': datetime.now(timezone.utc).isoformat(), 'python': platform.python_version(), 'git_head': revision(ROOT), 'wiki_git_head': revision(ROOT.parent / 'zpwiki'), 'fingerprints': content_fingerprints(questions)} def verify_freeze(manifest: dict, metadata: dict, graph: dict, *, limit: int, max_entities: int, audit: Path | None) -> None: if manifest['fingerprints'] != metadata['fingerprints']: raise RuntimeError('Frozen implementation, benchmark or inputs changed. TEST refused.') if manifest['graph'] != graph: raise RuntimeError('Neo4j graph changed since DEV freeze. TEST refused.') if manifest['parameters'] != {'limit': limit, 'max_entities': max_entities}: raise RuntimeError('Retrieval parameters differ from DEV freeze. TEST refused.') if audit is None or sha256_file(audit) != manifest['audit_sha256']: raise RuntimeError('Source audit differs from DEV freeze. TEST refused.')