"""Freeze the exact code, data, graph and parameters after complete DEV evaluation.""" from __future__ import annotations import argparse import json from datetime import datetime, timezone from pathlib import Path from .reproducibility import content_fingerprints, sha256_file def make_freeze(dev: dict, questions: list[dict], fingerprints: dict, audit: dict) -> dict: expected_ids = {q['id'] for q in questions if q['split'] == 'dev'} actual_ids = [row['id'] for row in dev['questions']] if dev['configuration']['split'] != 'dev' or set(actual_ids) != expected_ids or len(actual_ids) != len(expected_ids): raise RuntimeError('Freeze requires the complete DEV split, without duplicates.') if dev['summary']['error_count'] or any(row.get('error') for row in dev['questions']): raise RuntimeError('Freeze requires DEV without runtime errors.') if dev['run_metadata']['fingerprints'] != fingerprints: raise RuntimeError('Code or data changed after DEV; rerun DEV before freezing.') if audit['dataset_sha256'] != fingerprints['dataset_sha256'] or audit['inputs'] != fingerprints['inputs']: raise RuntimeError('Source audit does not match the benchmark and inputs.') if len(audit['questions']) != len(questions) or {r['id'] for r in audit['questions']} != {q['id'] for q in questions}: raise RuntimeError('Source audit must cover every question exactly once.') schema3 = any(question.get('graph', {}).get('schema_version') == 3 for question in questions) partition = None if schema3: if audit.get('source_corpus') != fingerprints.get('source_corpus'): raise RuntimeError('Independent audit does not match original Markdown sources.') if any(not row.get('source_consistent') for row in audit['questions']): raise RuntimeError('Resolve source-inconsistent annotations before freezing v2.') profiles = {split: {profile for question in questions if question['split'] == split for profile in question.get('graph', {}).get('source_profiles', [])} for split in ('dev', 'test')} if profiles['dev'] & profiles['test']: raise RuntimeError('DEV/TEST source profiles overlap; independent holdout refused.') if not profiles['dev'] or not profiles['test']: raise RuntimeError('Primary v2 freeze requires declared DEV and TEST source profiles.') partition = {'method': 'disjoint source profiles', 'dev_profile_count': len(profiles['dev']), 'test_profile_count': len(profiles['test']), 'overlap': 0} return {'schema_version': 2 if schema3 else 1, 'partition': partition, 'frozen_at_utc': datetime.now(timezone.utc).isoformat(), 'fingerprints': fingerprints, 'graph': dev['run_metadata']['graph'], 'parameters': {key: dev['configuration'][key] for key in ['limit', 'max_entities']}, 'dev_summary': dev['summary'], 'source_audit_summary': audit['by_split'], 'policy': 'Report all questions including negatives. No exclusions or holdout tuning.'} def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--dev-results', type=Path, required=True) parser.add_argument('--questions', type=Path, default=Path(__file__).parent / 'graphrag_questions_v3.json') parser.add_argument('--audit', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() if args.output.exists(): parser.error('Freeze manifest already exists; choose a new version/output path.') manifest = make_freeze(json.loads(args.dev_results.read_text()), json.loads(args.questions.read_text()), content_fingerprints(args.questions), json.loads(args.audit.read_text())) manifest.update({'dev_results_sha256': sha256_file(args.dev_results), 'audit_sha256': sha256_file(args.audit)}) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open('x', encoding='utf-8') as output: output.write(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') print(f'Frozen GraphRAG: {args.output}') if __name__ == '__main__': main()