90 lines
4.5 KiB
Python
90 lines
4.5 KiB
Python
"""Content fingerprints used to freeze retrieval before holdout evaluation."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from tools.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def source_corpus_fingerprint(root: Path = ROOT) -> dict[str, Any]:
|
|
"""Bind independent expectations to the original Markdown, not graph output."""
|
|
wiki = Path(os.getenv('ZPWIKI_ROOT', str(root.parent / 'zpwiki')))
|
|
pages = wiki / 'pages'
|
|
if not pages.is_dir():
|
|
raise RuntimeError(f'Original ZP Wiki Markdown directory is missing: {pages}')
|
|
documents = {str(path.relative_to(wiki)): sha256_file(path)
|
|
for path in sorted(pages.rglob('README.md')) if path.is_file()}
|
|
serialized = json.dumps(documents, sort_keys=True, separators=(',', ':')).encode()
|
|
return {'sha256': hashlib.sha256(serialized).hexdigest(),
|
|
'document_count': len(documents), 'documents': documents}
|
|
|
|
|
|
def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]:
|
|
files = sorted((root / 'tools/graphrag').rglob('*.py'))
|
|
# requirements.txt and tools/ are present both in the checkout and image.
|
|
files.extend(root / name for name in [
|
|
'requirements.txt', 'tools/publication.py', 'tools/retrieval_contract.py',
|
|
'tools/index_state.py', 'tools/rag/common.py', 'tools/sync_indexes.py'])
|
|
return {
|
|
'implementation': {str(path.relative_to(root)): sha256_file(path) for path in files},
|
|
'dataset_sha256': sha256_file(questions),
|
|
'inputs': {name: sha256_file(root / name) for name in ['data/documents.json', 'data/chunks.json']},
|
|
'source_corpus': source_corpus_fingerprint(root),
|
|
'retrieval_policy': {'published_only': DEFAULT_PUBLISHED_ONLY,
|
|
'publication_policy': PUBLICATION_POLICY_VERSION,
|
|
'academic_preparation_is_work': False},
|
|
}
|
|
|
|
|
|
def graph_fingerprint(session: Any) -> dict[str, Any]:
|
|
nodes = session.run(
|
|
'MATCH (n) WHERE n.graph_scope = $scope '
|
|
'RETURN labels(n) AS labels, properties(n) AS properties', scope='zpwiki'
|
|
).data()
|
|
edges = session.run(
|
|
'MATCH (a)-[r]->(b) WHERE a.graph_scope = $scope AND b.graph_scope = $scope '
|
|
'RETURN labels(a) AS source_labels, coalesce(a.id, a.path, a.name) AS source, '
|
|
'type(r) AS type, properties(r) AS properties, labels(b) AS target_labels, '
|
|
'coalesce(b.id, b.path, b.name) AS target', scope='zpwiki'
|
|
).data()
|
|
# elementId and row order are deliberately excluded: they change on rebuild.
|
|
canonical = lambda rows: sorted(json.dumps(row, sort_keys=True, ensure_ascii=False, default=str) for row in rows)
|
|
content = json.dumps({'nodes': canonical(nodes), 'relationships': canonical(edges)}, ensure_ascii=False)
|
|
return {'sha256': hashlib.sha256(content.encode()).hexdigest(),
|
|
'nodes': len(nodes), 'relationships': len(edges)}
|
|
|
|
|
|
def run_metadata(questions: Path) -> dict[str, Any]:
|
|
def revision(path: Path) -> str | None:
|
|
result = subprocess.run(['git', '-C', str(path), 'rev-parse', 'HEAD'],
|
|
text=True, capture_output=True, check=False)
|
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
return {'started_at_utc': datetime.now(timezone.utc).isoformat(),
|
|
'python': platform.python_version(), 'git_head': revision(ROOT),
|
|
'wiki_git_head': revision(Path(os.getenv('ZPWIKI_ROOT', str(ROOT.parent / 'zpwiki')))),
|
|
'fingerprints': content_fingerprints(questions)}
|
|
|
|
|
|
def verify_freeze(manifest: dict, metadata: dict, graph: dict, *, limit: int, max_entities: int,
|
|
audit: Path | None) -> None:
|
|
if manifest['fingerprints'] != metadata['fingerprints']:
|
|
raise RuntimeError('Frozen implementation, benchmark or inputs changed. TEST refused.')
|
|
if manifest['graph'] != graph:
|
|
raise RuntimeError('Neo4j graph changed since DEV freeze. TEST refused.')
|
|
if manifest['parameters'] != {'limit': limit, 'max_entities': max_entities}:
|
|
raise RuntimeError('Retrieval parameters differ from DEV freeze. TEST refused.')
|
|
if audit is None or sha256_file(audit) != manifest['audit_sha256']:
|
|
raise RuntimeError('Source audit differs from DEV freeze. TEST refused.')
|