226 lines
5.4 KiB
JSON
226 lines
5.4 KiB
JSON
{
|
|
"method": "Known-question development selection.",
|
|
"graph_freeze_sha256": "9e97794200520d0f2288e452c9e186c0291a54c63d2c1382f693a9753e4081cf",
|
|
"audit_sha256": "7825d4f3e4e535eb5f49e975dfef363b9b813f1de5c97e2387e5c2f3e578e6a5",
|
|
"generation_files": {
|
|
"tools/answer_evaluation/evaluate_graph_answers.py": "d81504e8fe51598c6bd1b0d482f61b9b6f51f87ba3f38b03ae1bc584675a3b76",
|
|
"tools/answer_evaluation/graph_answer_prompt.txt": "fc9f2f8d52e70a8d67b9f4814b3f15e6be64bf88afe53ce4db4f037bde00e064",
|
|
"tools/answer_evaluation/answer_format.py": "86a27b4aebc6a5077e13794b81957a1cf063a1f3e070c583120ba53c1788d2f8",
|
|
"tools/answer_evaluation/context_policy.py": "0dc2a056ca0d886d7cc8248ed4632c910c5e35bebd03ef94573df41802297f47",
|
|
"tools/answer_evaluation/checked_generation.py": "cb821a65327424c19cbd996190b14e6aa30d9b909493f11716552f71249a0529",
|
|
"tools/answer_evaluation/render_answer.py": "609c534a37899b3febcea32d983da4b806c287de8f2ddf9bc0675e3280859002",
|
|
"tools/rag/evaluation/rag_runner.py": "031abf7d5b1a026eb199aa2a579bfb8baaa01175279d0018a364bfe50b76c3c1"
|
|
},
|
|
"model": "model120-fast",
|
|
"temperature": 0,
|
|
"workers": 3,
|
|
"dev_ids": [
|
|
"v7-dev-year-1899",
|
|
"v7-dev-year-1999",
|
|
"v7-dev-year-2100",
|
|
"v7-dev-year-9999",
|
|
"wide-001",
|
|
"wide-002",
|
|
"wide-041",
|
|
"wide-042",
|
|
"wide-080",
|
|
"wide-081",
|
|
"wide-082",
|
|
"wide-104",
|
|
"wide-113",
|
|
"wide-114",
|
|
"wide-116",
|
|
"wide-119",
|
|
"wide-129",
|
|
"wide-137",
|
|
"wide-138",
|
|
"wide-139",
|
|
"wide-141",
|
|
"wide-142",
|
|
"wide-143",
|
|
"wide-145",
|
|
"wide-155",
|
|
"wide-159",
|
|
"wide-161",
|
|
"wide-163",
|
|
"wide-165",
|
|
"wide-169",
|
|
"wide-170",
|
|
"wide-178",
|
|
"wide-181",
|
|
"wide-193",
|
|
"wide-194",
|
|
"wide-209",
|
|
"wide-217",
|
|
"wide-218",
|
|
"wide-232",
|
|
"wide-236",
|
|
"wide-239",
|
|
"wide-249",
|
|
"wide-250",
|
|
"wide-283",
|
|
"wide-289",
|
|
"wide-290"
|
|
],
|
|
"test_ids": [
|
|
"fresh7-001",
|
|
"fresh7-002",
|
|
"fresh7-003",
|
|
"fresh7-004",
|
|
"fresh7-005",
|
|
"fresh7-006",
|
|
"fresh7-007",
|
|
"fresh7-008",
|
|
"fresh7-009",
|
|
"fresh7-010",
|
|
"fresh7-011",
|
|
"fresh7-012",
|
|
"fresh7-013",
|
|
"fresh7-014",
|
|
"fresh7-015",
|
|
"fresh7-016",
|
|
"fresh7-017",
|
|
"fresh7-018",
|
|
"fresh7-019",
|
|
"fresh7-020",
|
|
"fresh7-021",
|
|
"fresh7-022",
|
|
"fresh7-023",
|
|
"fresh7-024",
|
|
"fresh7-025",
|
|
"fresh7-026",
|
|
"fresh7-027",
|
|
"fresh7-028",
|
|
"fresh7-029",
|
|
"fresh7-030",
|
|
"fresh7-031",
|
|
"fresh7-032",
|
|
"fresh7-033",
|
|
"fresh7-034",
|
|
"fresh7-035",
|
|
"fresh7-036",
|
|
"fresh7-037",
|
|
"fresh7-038",
|
|
"fresh7-039",
|
|
"fresh7-040",
|
|
"fresh7-041",
|
|
"fresh7-042",
|
|
"fresh7-043",
|
|
"fresh7-044",
|
|
"fresh7-045",
|
|
"fresh7-046",
|
|
"fresh7-047",
|
|
"fresh7-048",
|
|
"fresh7-049",
|
|
"fresh7-050",
|
|
"fresh7-051",
|
|
"fresh7-052",
|
|
"fresh7-053",
|
|
"fresh7-054",
|
|
"fresh7-055",
|
|
"fresh7-056",
|
|
"fresh7-057",
|
|
"fresh7-058",
|
|
"fresh7-059",
|
|
"fresh7-060",
|
|
"fresh7-061",
|
|
"fresh7-062",
|
|
"fresh7-063",
|
|
"fresh7-064",
|
|
"fresh7-065",
|
|
"fresh7-066",
|
|
"fresh7-067",
|
|
"fresh7-068",
|
|
"fresh7-069",
|
|
"fresh7-070",
|
|
"fresh7-071",
|
|
"fresh7-072",
|
|
"fresh7-073",
|
|
"fresh7-074",
|
|
"fresh7-075",
|
|
"fresh7-076",
|
|
"fresh7-077",
|
|
"fresh7-078",
|
|
"fresh7-079",
|
|
"fresh7-080"
|
|
],
|
|
"dev_count": 46,
|
|
"test_count": 80,
|
|
"target_accuracy": 0.92,
|
|
"evaluation_kind": "development",
|
|
"rendering": "Verified structured claims rendered deterministically; raw model outputs preserved.",
|
|
"selected_split": "dev",
|
|
"selected_ids": [
|
|
"wide-001",
|
|
"wide-002",
|
|
"wide-041",
|
|
"wide-042",
|
|
"wide-080",
|
|
"wide-081",
|
|
"wide-082",
|
|
"wide-104",
|
|
"wide-113",
|
|
"wide-114",
|
|
"wide-116",
|
|
"wide-119",
|
|
"wide-129",
|
|
"wide-137",
|
|
"wide-138",
|
|
"wide-139",
|
|
"wide-141",
|
|
"wide-142",
|
|
"wide-143",
|
|
"wide-145",
|
|
"wide-155",
|
|
"wide-159",
|
|
"wide-161",
|
|
"wide-163",
|
|
"wide-165",
|
|
"wide-169",
|
|
"wide-170",
|
|
"wide-178",
|
|
"wide-181",
|
|
"wide-193",
|
|
"wide-194",
|
|
"wide-209",
|
|
"wide-217",
|
|
"wide-218",
|
|
"wide-232",
|
|
"wide-236",
|
|
"wide-239",
|
|
"wide-249",
|
|
"wide-250",
|
|
"wide-283",
|
|
"wide-289",
|
|
"wide-290",
|
|
"v7-dev-year-1999",
|
|
"v7-dev-year-1899",
|
|
"v7-dev-year-2100",
|
|
"v7-dev-year-9999"
|
|
],
|
|
"selected_count": 46,
|
|
"selected_dev_overlap": 46,
|
|
"corrective_request_limit": 1,
|
|
"format_repair_limit": 1,
|
|
"repair_policy": "At most one total corrective request for format, citations or retrieval-context violations. Same question/context, no gold; retain all attempts.",
|
|
"dev_results_sha256": null,
|
|
"policy": "Evaluate every declared fresh TEST case once; retain every failure; do not tune on this TEST.",
|
|
"selected_categories": {
|
|
"shared_keyword_pair": 2,
|
|
"same_academic_year_pair": 3,
|
|
"same_start_year_pair": 3,
|
|
"person_title_year": 5,
|
|
"person_keyword_title": 12,
|
|
"document_author_category_multi_hop": 4,
|
|
"person_multiple_categories": 3,
|
|
"candidate_set_aggregation": 5,
|
|
"conjunctive_disambiguation": 3,
|
|
"negative_lookup": 6
|
|
},
|
|
"limitations": [
|
|
"Known development questions; not a holdout.",
|
|
"Harness supplies graph context; native OpenWebUI tool choice and browser UI are outside this experiment.",
|
|
"Checks structured factual claims, their source support/citations and required prose values; not a general semantic judge of arbitrary prose."
|
|
]
|
|
}
|