"""Content fingerprints used to freeze retrieval before holdout evaluation.""" from __future__ import annotations import hashlib import json import os import platform import subprocess from datetime import datetime, timezone from pathlib import Path from typing import Any from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION ROOT = Path(__file__).resolve().parents[3] def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def source_corpus_fingerprint(root: Path = ROOT) -> dict[str, Any]: """Bind independent expectations to the original Markdown, not graph output.""" wiki = Path(os.getenv('ZPWIKI_ROOT', str(root.parent / 'zpwiki'))) pages = wiki / 'pages' if not pages.is_dir(): raise RuntimeError(f'Original ZP Wiki Markdown directory is missing: {pages}') documents = {str(path.relative_to(wiki)): sha256_file(path) for path in sorted(pages.rglob('README.md')) if path.is_file()} serialized = json.dumps(documents, sort_keys=True, separators=(',', ':')).encode() return {'sha256': hashlib.sha256(serialized).hexdigest(), 'document_count': len(documents), 'documents': documents} def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: files = sorted((root / 'tools/graphrag').rglob('*.py')) # Shared production packages moved out of tools/. Bind their complete code # too, so a freeze cannot silently overlook new runtime/delivery helpers. for package in ('answers', 'retrieval', 'runtime', 'sync'): files.extend(sorted((root / 'tools' / package).rglob('*.py'))) # Deployment requirements and tools/ exist in both checkout and image. files.extend(root / name for name in [ 'deployment/requirements.txt', 'tools/retrieval/publication.py', 'tools/retrieval/retrieval_contract.py', 'tools/sync/index_state.py', 'tools/rag/common.py', 'tools/sync/sync_indexes.py', 'tools/rag/rag_utils.py', 'tools/answer_evaluation/context_policy.py', 'tools/answer_evaluation/checked_generation.py', 'tools/answer_evaluation/answer_format.py', 'tools/answer_evaluation/render_answer.py', 'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py']) files = sorted(set(files)) return { 'implementation': {str(path.relative_to(root)): sha256_file(path) for path in files}, 'dataset_sha256': sha256_file(questions), 'inputs': {name: sha256_file(root / name) for name in ['data/documents.json', 'data/chunks.json']}, 'source_corpus': source_corpus_fingerprint(root), 'retrieval_policy': {'published_only': DEFAULT_PUBLISHED_ONLY, 'publication_policy': PUBLICATION_POLICY_VERSION, 'academic_preparation_is_work': False}, } def graph_fingerprint(session: Any) -> dict[str, Any]: nodes = session.run( 'MATCH (n) WHERE n.graph_scope = $scope ' 'RETURN labels(n) AS labels, properties(n) AS properties', scope='zpwiki' ).data() edges = session.run( 'MATCH (a)-[r]->(b) WHERE a.graph_scope = $scope AND b.graph_scope = $scope ' 'RETURN labels(a) AS source_labels, coalesce(a.id, a.path, a.name) AS source, ' 'type(r) AS type, properties(r) AS properties, labels(b) AS target_labels, ' 'coalesce(b.id, b.path, b.name) AS target', scope='zpwiki' ).data() # elementId and row order are deliberately excluded: they change on rebuild. canonical = lambda rows: sorted(json.dumps(row, sort_keys=True, ensure_ascii=False, default=str) for row in rows) content = json.dumps({'nodes': canonical(nodes), 'relationships': canonical(edges)}, ensure_ascii=False) return {'sha256': hashlib.sha256(content.encode()).hexdigest(), 'nodes': len(nodes), 'relationships': len(edges)} def run_metadata(questions: Path) -> dict[str, Any]: def revision(path: Path) -> str | None: result = subprocess.run(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True, capture_output=True, check=False) return result.stdout.strip() if result.returncode == 0 else None return {'started_at_utc': datetime.now(timezone.utc).isoformat(), 'python': platform.python_version(), 'git_head': revision(ROOT), 'wiki_git_head': revision(Path(os.getenv('ZPWIKI_ROOT', str(ROOT.parent / 'zpwiki')))), 'fingerprints': content_fingerprints(questions)} def verify_freeze(manifest: dict, metadata: dict, graph: dict, *, limit: int, max_entities: int, audit: Path | None) -> None: if manifest['fingerprints'] != metadata['fingerprints']: raise RuntimeError('Frozen implementation, benchmark or inputs changed. TEST refused.') if manifest['graph'] != graph: raise RuntimeError('Neo4j graph changed since DEV freeze. TEST refused.') if manifest['parameters'] != {'limit': limit, 'max_entities': max_entities}: raise RuntimeError('Retrieval parameters differ from DEV freeze. TEST refused.') if audit is None or sha256_file(audit) != manifest['audit_sha256']: raise RuntimeError('Source audit differs from DEV freeze. TEST refused.')