50 lines
2.8 KiB
Python
50 lines
2.8 KiB
Python
"""Freeze the exact code, data, graph and parameters after complete DEV evaluation."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from .reproducibility import content_fingerprints, sha256_file
|
|
|
|
|
|
def make_freeze(dev: dict, questions: list[dict], fingerprints: dict, audit: dict) -> dict:
|
|
expected_ids = {q['id'] for q in questions if q['split'] == 'dev'}
|
|
actual_ids = [row['id'] for row in dev['questions']]
|
|
if dev['configuration']['split'] != 'dev' or set(actual_ids) != expected_ids or len(actual_ids) != len(expected_ids):
|
|
raise RuntimeError('Freeze requires the complete DEV split, without duplicates.')
|
|
if dev['summary']['error_count'] or any(row.get('error') for row in dev['questions']):
|
|
raise RuntimeError('Freeze requires DEV without runtime errors.')
|
|
if dev['run_metadata']['fingerprints'] != fingerprints:
|
|
raise RuntimeError('Code or data changed after DEV; rerun DEV before freezing.')
|
|
if audit['dataset_sha256'] != fingerprints['dataset_sha256'] or audit['inputs'] != fingerprints['inputs']:
|
|
raise RuntimeError('Source audit does not match the benchmark and inputs.')
|
|
if len(audit['questions']) != len(questions) or {r['id'] for r in audit['questions']} != {q['id'] for q in questions}:
|
|
raise RuntimeError('Source audit must cover every question exactly once.')
|
|
return {'schema_version': 1, 'frozen_at_utc': datetime.now(timezone.utc).isoformat(),
|
|
'fingerprints': fingerprints, 'graph': dev['run_metadata']['graph'],
|
|
'parameters': {key: dev['configuration'][key] for key in ['limit', 'max_entities']},
|
|
'dev_summary': dev['summary'], 'source_audit_summary': audit['by_split'],
|
|
'policy': 'Report all questions. Source-consistent subset is supplementary. No holdout tuning.'}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--dev-results', type=Path, required=True)
|
|
parser.add_argument('--questions', type=Path, default=Path(__file__).parent / 'graphrag_questions.json')
|
|
parser.add_argument('--audit', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
manifest = make_freeze(json.loads(args.dev_results.read_text()), json.loads(args.questions.read_text()),
|
|
content_fingerprints(args.questions), json.loads(args.audit.read_text()))
|
|
manifest.update({'dev_results_sha256': sha256_file(args.dev_results),
|
|
'audit_sha256': sha256_file(args.audit)})
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
|
print(f'Frozen GraphRAG: {args.output}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|