{
  "method": "Known-question development selection.",
  "graph_freeze_sha256": "9e97794200520d0f2288e452c9e186c0291a54c63d2c1382f693a9753e4081cf",
  "audit_sha256": "7825d4f3e4e535eb5f49e975dfef363b9b813f1de5c97e2387e5c2f3e578e6a5",
  "generation_files": {
    "tools/answer_evaluation/evaluate_graph_answers.py": "d81504e8fe51598c6bd1b0d482f61b9b6f51f87ba3f38b03ae1bc584675a3b76",
    "tools/answer_evaluation/graph_answer_prompt.txt": "fc9f2f8d52e70a8d67b9f4814b3f15e6be64bf88afe53ce4db4f037bde00e064",
    "tools/answer_evaluation/answer_format.py": "86a27b4aebc6a5077e13794b81957a1cf063a1f3e070c583120ba53c1788d2f8",
    "tools/answer_evaluation/context_policy.py": "0dc2a056ca0d886d7cc8248ed4632c910c5e35bebd03ef94573df41802297f47",
    "tools/answer_evaluation/checked_generation.py": "cb821a65327424c19cbd996190b14e6aa30d9b909493f11716552f71249a0529",
    "tools/answer_evaluation/render_answer.py": "609c534a37899b3febcea32d983da4b806c287de8f2ddf9bc0675e3280859002",
    "tools/rag/evaluation/rag_runner.py": "031abf7d5b1a026eb199aa2a579bfb8baaa01175279d0018a364bfe50b76c3c1"
  },
  "model": "model120-fast",
  "temperature": 0,
  "workers": 3,
  "dev_ids": [
    "v7-dev-year-1899",
    "v7-dev-year-1999",
    "v7-dev-year-2100",
    "v7-dev-year-9999",
    "wide-001",
    "wide-002",
    "wide-041",
    "wide-042",
    "wide-080",
    "wide-081",
    "wide-082",
    "wide-104",
    "wide-113",
    "wide-114",
    "wide-116",
    "wide-119",
    "wide-129",
    "wide-137",
    "wide-138",
    "wide-139",
    "wide-141",
    "wide-142",
    "wide-143",
    "wide-145",
    "wide-155",
    "wide-159",
    "wide-161",
    "wide-163",
    "wide-165",
    "wide-169",
    "wide-170",
    "wide-178",
    "wide-181",
    "wide-193",
    "wide-194",
    "wide-209",
    "wide-217",
    "wide-218",
    "wide-232",
    "wide-236",
    "wide-239",
    "wide-249",
    "wide-250",
    "wide-283",
    "wide-289",
    "wide-290"
  ],
  "test_ids": [
    "fresh7-001",
    "fresh7-002",
    "fresh7-003",
    "fresh7-004",
    "fresh7-005",
    "fresh7-006",
    "fresh7-007",
    "fresh7-008",
    "fresh7-009",
    "fresh7-010",
    "fresh7-011",
    "fresh7-012",
    "fresh7-013",
    "fresh7-014",
    "fresh7-015",
    "fresh7-016",
    "fresh7-017",
    "fresh7-018",
    "fresh7-019",
    "fresh7-020",
    "fresh7-021",
    "fresh7-022",
    "fresh7-023",
    "fresh7-024",
    "fresh7-025",
    "fresh7-026",
    "fresh7-027",
    "fresh7-028",
    "fresh7-029",
    "fresh7-030",
    "fresh7-031",
    "fresh7-032",
    "fresh7-033",
    "fresh7-034",
    "fresh7-035",
    "fresh7-036",
    "fresh7-037",
    "fresh7-038",
    "fresh7-039",
    "fresh7-040",
    "fresh7-041",
    "fresh7-042",
    "fresh7-043",
    "fresh7-044",
    "fresh7-045",
    "fresh7-046",
    "fresh7-047",
    "fresh7-048",
    "fresh7-049",
    "fresh7-050",
    "fresh7-051",
    "fresh7-052",
    "fresh7-053",
    "fresh7-054",
    "fresh7-055",
    "fresh7-056",
    "fresh7-057",
    "fresh7-058",
    "fresh7-059",
    "fresh7-060",
    "fresh7-061",
    "fresh7-062",
    "fresh7-063",
    "fresh7-064",
    "fresh7-065",
    "fresh7-066",
    "fresh7-067",
    "fresh7-068",
    "fresh7-069",
    "fresh7-070",
    "fresh7-071",
    "fresh7-072",
    "fresh7-073",
    "fresh7-074",
    "fresh7-075",
    "fresh7-076",
    "fresh7-077",
    "fresh7-078",
    "fresh7-079",
    "fresh7-080"
  ],
  "dev_count": 46,
  "test_count": 80,
  "target_accuracy": 0.92,
  "evaluation_kind": "development",
  "rendering": "Verified structured claims rendered deterministically; raw model outputs preserved.",
  "selected_split": "dev",
  "selected_ids": [
    "wide-001",
    "wide-002",
    "wide-041",
    "wide-042",
    "wide-080",
    "wide-081",
    "wide-082",
    "wide-104",
    "wide-113",
    "wide-114",
    "wide-116",
    "wide-119",
    "wide-129",
    "wide-137",
    "wide-138",
    "wide-139",
    "wide-141",
    "wide-142",
    "wide-143",
    "wide-145",
    "wide-155",
    "wide-159",
    "wide-161",
    "wide-163",
    "wide-165",
    "wide-169",
    "wide-170",
    "wide-178",
    "wide-181",
    "wide-193",
    "wide-194",
    "wide-209",
    "wide-217",
    "wide-218",
    "wide-232",
    "wide-236",
    "wide-239",
    "wide-249",
    "wide-250",
    "wide-283",
    "wide-289",
    "wide-290",
    "v7-dev-year-1999",
    "v7-dev-year-1899",
    "v7-dev-year-2100",
    "v7-dev-year-9999"
  ],
  "selected_count": 46,
  "selected_dev_overlap": 46,
  "corrective_request_limit": 1,
  "format_repair_limit": 1,
  "repair_policy": "At most one total corrective request for format, citations or retrieval-context violations. Same question/context, no gold; retain all attempts.",
  "dev_results_sha256": null,
  "policy": "Evaluate every declared fresh TEST case once; retain every failure; do not tune on this TEST.",
  "selected_categories": {
    "shared_keyword_pair": 2,
    "same_academic_year_pair": 3,
    "same_start_year_pair": 3,
    "person_title_year": 5,
    "person_keyword_title": 12,
    "document_author_category_multi_hop": 4,
    "person_multiple_categories": 3,
    "candidate_set_aggregation": 5,
    "conjunctive_disambiguation": 3,
    "negative_lookup": 6
  },
  "limitations": [
    "Known development questions; not a holdout.",
    "Harness supplies graph context; native OpenWebUI tool choice and browser UI are outside this experiment.",
    "Checks structured factual claims, their source support/citations and required prose values; not a general semantic judge of arbitrary prose."
  ]
}