171 lines
7.5 KiB
Python
171 lines
7.5 KiB
Python
"""Freeze the exact code, data, graph and parameters after complete DEV evaluation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from .reproducibility import content_fingerprints, sha256_file
|
|
|
|
|
|
def make_freeze(
|
|
dev: dict,
|
|
questions: list[dict],
|
|
fingerprints: dict,
|
|
audit: dict,
|
|
*,
|
|
purpose: str = "independent_holdout",
|
|
) -> dict:
|
|
if purpose not in {"independent_holdout", "historical_regression"}:
|
|
raise ValueError("Unknown evaluation purpose.")
|
|
expected_ids = {q["id"] for q in questions if q["split"] == "dev"}
|
|
actual_ids = [row["id"] for row in dev["questions"]]
|
|
if (
|
|
dev["configuration"]["split"] != "dev"
|
|
or set(actual_ids) != expected_ids
|
|
or len(actual_ids) != len(expected_ids)
|
|
):
|
|
raise RuntimeError("Freeze requires the complete DEV split, without duplicates.")
|
|
if dev["summary"]["error_count"] or any(row.get("error") for row in dev["questions"]):
|
|
raise RuntimeError("Freeze requires DEV without runtime errors.")
|
|
if dev["run_metadata"]["fingerprints"] != fingerprints:
|
|
raise RuntimeError("Code or data changed after DEV; rerun DEV before freezing.")
|
|
if fingerprints.get("retrieval_policy") and any(
|
|
row.get("source_checked") is not True or row.get("all_facts_supported") is not True
|
|
for row in dev["questions"]
|
|
):
|
|
raise RuntimeError(
|
|
"Every DEV result must pass independent validation of all returned facts."
|
|
)
|
|
if (
|
|
audit["dataset_sha256"] != fingerprints["dataset_sha256"]
|
|
or audit["inputs"] != fingerprints["inputs"]
|
|
):
|
|
raise RuntimeError("Source audit does not match the benchmark and inputs.")
|
|
if len(audit["questions"]) != len(questions) or {r["id"] for r in audit["questions"]} != {
|
|
q["id"] for q in questions
|
|
}:
|
|
raise RuntimeError("Source audit must cover every question exactly once.")
|
|
schema3 = any(question.get("graph", {}).get("schema_version") == 3 for question in questions)
|
|
partition = None
|
|
if schema3:
|
|
if audit.get("source_corpus") != fingerprints.get("source_corpus"):
|
|
raise RuntimeError("Independent audit does not match original Markdown sources.")
|
|
if any(not row.get("source_consistent") for row in audit["questions"]):
|
|
raise RuntimeError("Resolve source-inconsistent annotations before freezing v2.")
|
|
profiles = {
|
|
split: {
|
|
profile
|
|
for question in questions
|
|
if question["split"] == split
|
|
for profile in question.get("graph", {}).get("source_profiles", [])
|
|
}
|
|
for split in ("dev", "test")
|
|
}
|
|
policy = audit.get("partition_policy", {})
|
|
if purpose == "historical_regression":
|
|
partition = {
|
|
"method": "Previously exposed questions; regression, not independent TEST",
|
|
"dev_profile_count": len(profiles["dev"]),
|
|
"test_profile_count": len(profiles["test"]),
|
|
"source_profile_overlap": len(profiles["dev"] & profiles["test"]),
|
|
}
|
|
elif policy.get("method") == "disjoint logical query groups":
|
|
# This mode explicitly reports known profiles. It must never be
|
|
# presented as the stronger, historic unseen-profile experiment.
|
|
from .benchmark_v4 import logical_group
|
|
from .source_oracle import norm
|
|
|
|
groups = {
|
|
split: {logical_group(q) for q in questions if q["split"] == split}
|
|
for split in ("dev", "test")
|
|
}
|
|
if any(q["graph"].get("partition_group") != logical_group(q) for q in questions):
|
|
raise RuntimeError(
|
|
"Declared logical group does not match the question constraints."
|
|
)
|
|
if not groups["dev"] or not groups["test"] or groups["dev"] & groups["test"]:
|
|
raise RuntimeError("DEV/TEST logical groups overlap or are missing.")
|
|
exposed = set(policy.get("previously_exposed_groups", []))
|
|
if not exposed or groups["test"] & exposed:
|
|
raise RuntimeError("TEST contains previously exposed logical groups.")
|
|
texts = {
|
|
split: {norm(q["question"]) for q in questions if q["split"] == split}
|
|
for split in ("dev", "test")
|
|
}
|
|
if texts["dev"] & texts["test"]:
|
|
raise RuntimeError("DEV/TEST question texts overlap.")
|
|
partition = {
|
|
"method": policy["method"],
|
|
"logical_group_overlap": 0,
|
|
"previously_exposed_test_groups": 0,
|
|
"dev_group_count": len(groups["dev"]),
|
|
"test_group_count": len(groups["test"]),
|
|
"dev_profile_count": len(profiles["dev"]),
|
|
"test_profile_count": len(profiles["test"]),
|
|
"source_profile_overlap": len(profiles["dev"] & profiles["test"]),
|
|
"limitations": policy["limitations"],
|
|
}
|
|
elif profiles["dev"] & profiles["test"]:
|
|
raise RuntimeError("DEV/TEST source profiles overlap; independent holdout refused.")
|
|
if not profiles["dev"] or not profiles["test"]:
|
|
raise RuntimeError("Primary v2 freeze requires declared DEV and TEST source profiles.")
|
|
if partition is None:
|
|
partition = {
|
|
"method": "disjoint source profiles",
|
|
"dev_profile_count": len(profiles["dev"]),
|
|
"test_profile_count": len(profiles["test"]),
|
|
"overlap": 0,
|
|
}
|
|
return {
|
|
"schema_version": 2 if schema3 else 1,
|
|
"evaluation_kind": purpose,
|
|
"partition": partition,
|
|
"frozen_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"fingerprints": fingerprints,
|
|
"graph": dev["run_metadata"]["graph"],
|
|
"parameters": {key: dev["configuration"][key] for key in ["limit", "max_entities"]},
|
|
"dev_summary": dev["summary"],
|
|
"source_audit_summary": audit["by_split"],
|
|
"policy": "Report all questions including negatives. No exclusions or holdout tuning.",
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dev-results", type=Path, required=True)
|
|
parser.add_argument("--questions", type=Path, required=True)
|
|
parser.add_argument("--audit", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument(
|
|
"--purpose",
|
|
choices=["independent_holdout", "historical_regression"],
|
|
default="independent_holdout",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.output.exists():
|
|
parser.error("Freeze manifest already exists; choose a new version/output path.")
|
|
manifest = make_freeze(
|
|
json.loads(args.dev_results.read_text()),
|
|
json.loads(args.questions.read_text()),
|
|
content_fingerprints(args.questions),
|
|
json.loads(args.audit.read_text()),
|
|
purpose=args.purpose,
|
|
)
|
|
manifest.update(
|
|
{
|
|
"dev_results_sha256": sha256_file(args.dev_results),
|
|
"audit_sha256": sha256_file(args.audit),
|
|
}
|
|
)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.output.open("x", encoding="utf-8") as output:
|
|
output.write(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
|
|
print(f"Frozen GraphRAG: {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|