500 lines
9.8 KiB
JSON
500 lines
9.8 KiB
JSON
{
|
|
"method": "Previously exposed LLM questions; regression, not independent TEST.",
|
|
"graph_freeze_sha256": "9e97794200520d0f2288e452c9e186c0291a54c63d2c1382f693a9753e4081cf",
|
|
"audit_sha256": "7825d4f3e4e535eb5f49e975dfef363b9b813f1de5c97e2387e5c2f3e578e6a5",
|
|
"generation_files": {
|
|
"tools/answer_evaluation/evaluate_graph_answers.py": "d81504e8fe51598c6bd1b0d482f61b9b6f51f87ba3f38b03ae1bc584675a3b76",
|
|
"tools/answer_evaluation/graph_answer_prompt.txt": "fc9f2f8d52e70a8d67b9f4814b3f15e6be64bf88afe53ce4db4f037bde00e064",
|
|
"tools/answer_evaluation/answer_format.py": "86a27b4aebc6a5077e13794b81957a1cf063a1f3e070c583120ba53c1788d2f8",
|
|
"tools/answer_evaluation/context_policy.py": "0dc2a056ca0d886d7cc8248ed4632c910c5e35bebd03ef94573df41802297f47",
|
|
"tools/answer_evaluation/checked_generation.py": "cb821a65327424c19cbd996190b14e6aa30d9b909493f11716552f71249a0529",
|
|
"tools/answer_evaluation/render_answer.py": "609c534a37899b3febcea32d983da4b806c287de8f2ddf9bc0675e3280859002",
|
|
"tools/rag/evaluation/rag_runner.py": "031abf7d5b1a026eb199aa2a579bfb8baaa01175279d0018a364bfe50b76c3c1"
|
|
},
|
|
"model": "model120-fast",
|
|
"temperature": 0,
|
|
"workers": 3,
|
|
"dev_ids": [
|
|
"v7-dev-year-1899",
|
|
"v7-dev-year-1999",
|
|
"v7-dev-year-2100",
|
|
"v7-dev-year-9999",
|
|
"wide-001",
|
|
"wide-002",
|
|
"wide-041",
|
|
"wide-042",
|
|
"wide-080",
|
|
"wide-081",
|
|
"wide-082",
|
|
"wide-104",
|
|
"wide-113",
|
|
"wide-114",
|
|
"wide-116",
|
|
"wide-119",
|
|
"wide-129",
|
|
"wide-137",
|
|
"wide-138",
|
|
"wide-139",
|
|
"wide-141",
|
|
"wide-142",
|
|
"wide-143",
|
|
"wide-145",
|
|
"wide-155",
|
|
"wide-159",
|
|
"wide-161",
|
|
"wide-163",
|
|
"wide-165",
|
|
"wide-169",
|
|
"wide-170",
|
|
"wide-178",
|
|
"wide-181",
|
|
"wide-193",
|
|
"wide-194",
|
|
"wide-209",
|
|
"wide-217",
|
|
"wide-218",
|
|
"wide-232",
|
|
"wide-236",
|
|
"wide-239",
|
|
"wide-249",
|
|
"wide-250",
|
|
"wide-283",
|
|
"wide-289",
|
|
"wide-290"
|
|
],
|
|
"test_ids": [
|
|
"fresh7-001",
|
|
"fresh7-002",
|
|
"fresh7-003",
|
|
"fresh7-004",
|
|
"fresh7-005",
|
|
"fresh7-006",
|
|
"fresh7-007",
|
|
"fresh7-008",
|
|
"fresh7-009",
|
|
"fresh7-010",
|
|
"fresh7-011",
|
|
"fresh7-012",
|
|
"fresh7-013",
|
|
"fresh7-014",
|
|
"fresh7-015",
|
|
"fresh7-016",
|
|
"fresh7-017",
|
|
"fresh7-018",
|
|
"fresh7-019",
|
|
"fresh7-020",
|
|
"fresh7-021",
|
|
"fresh7-022",
|
|
"fresh7-023",
|
|
"fresh7-024",
|
|
"fresh7-025",
|
|
"fresh7-026",
|
|
"fresh7-027",
|
|
"fresh7-028",
|
|
"fresh7-029",
|
|
"fresh7-030",
|
|
"fresh7-031",
|
|
"fresh7-032",
|
|
"fresh7-033",
|
|
"fresh7-034",
|
|
"fresh7-035",
|
|
"fresh7-036",
|
|
"fresh7-037",
|
|
"fresh7-038",
|
|
"fresh7-039",
|
|
"fresh7-040",
|
|
"fresh7-041",
|
|
"fresh7-042",
|
|
"fresh7-043",
|
|
"fresh7-044",
|
|
"fresh7-045",
|
|
"fresh7-046",
|
|
"fresh7-047",
|
|
"fresh7-048",
|
|
"fresh7-049",
|
|
"fresh7-050",
|
|
"fresh7-051",
|
|
"fresh7-052",
|
|
"fresh7-053",
|
|
"fresh7-054",
|
|
"fresh7-055",
|
|
"fresh7-056",
|
|
"fresh7-057",
|
|
"fresh7-058",
|
|
"fresh7-059",
|
|
"fresh7-060",
|
|
"fresh7-061",
|
|
"fresh7-062",
|
|
"fresh7-063",
|
|
"fresh7-064",
|
|
"fresh7-065",
|
|
"fresh7-066",
|
|
"fresh7-067",
|
|
"fresh7-068",
|
|
"fresh7-069",
|
|
"fresh7-070",
|
|
"fresh7-071",
|
|
"fresh7-072",
|
|
"fresh7-073",
|
|
"fresh7-074",
|
|
"fresh7-075",
|
|
"fresh7-076",
|
|
"fresh7-077",
|
|
"fresh7-078",
|
|
"fresh7-079",
|
|
"fresh7-080"
|
|
],
|
|
"dev_count": 46,
|
|
"test_count": 80,
|
|
"target_accuracy": 0.92,
|
|
"evaluation_kind": "historical_regression",
|
|
"rendering": "Verified structured claims rendered deterministically; raw model outputs preserved.",
|
|
"selected_split": "regression",
|
|
"selected_ids": [
|
|
"wide-001",
|
|
"wide-002",
|
|
"wide-003",
|
|
"wide-004",
|
|
"wide-005",
|
|
"wide-006",
|
|
"wide-007",
|
|
"wide-008",
|
|
"wide-009",
|
|
"wide-010",
|
|
"wide-011",
|
|
"wide-012",
|
|
"wide-013",
|
|
"wide-014",
|
|
"wide-015",
|
|
"wide-016",
|
|
"wide-017",
|
|
"wide-018",
|
|
"wide-019",
|
|
"wide-020",
|
|
"wide-021",
|
|
"wide-022",
|
|
"wide-023",
|
|
"wide-024",
|
|
"wide-025",
|
|
"wide-026",
|
|
"wide-027",
|
|
"wide-028",
|
|
"wide-029",
|
|
"wide-030",
|
|
"wide-031",
|
|
"wide-032",
|
|
"wide-033",
|
|
"wide-034",
|
|
"wide-035",
|
|
"wide-036",
|
|
"wide-037",
|
|
"wide-038",
|
|
"wide-039",
|
|
"wide-040",
|
|
"wide-041",
|
|
"wide-042",
|
|
"wide-043",
|
|
"wide-044",
|
|
"wide-045",
|
|
"wide-046",
|
|
"wide-047",
|
|
"wide-048",
|
|
"wide-049",
|
|
"wide-050",
|
|
"wide-051",
|
|
"wide-052",
|
|
"wide-053",
|
|
"wide-054",
|
|
"wide-055",
|
|
"wide-056",
|
|
"wide-057",
|
|
"wide-058",
|
|
"wide-059",
|
|
"wide-060",
|
|
"wide-061",
|
|
"wide-062",
|
|
"wide-063",
|
|
"wide-064",
|
|
"wide-065",
|
|
"wide-066",
|
|
"wide-067",
|
|
"wide-068",
|
|
"wide-069",
|
|
"wide-070",
|
|
"wide-071",
|
|
"wide-072",
|
|
"wide-073",
|
|
"wide-074",
|
|
"wide-075",
|
|
"wide-076",
|
|
"wide-077",
|
|
"wide-078",
|
|
"wide-079",
|
|
"wide-080",
|
|
"wide-081",
|
|
"wide-082",
|
|
"wide-083",
|
|
"wide-084",
|
|
"wide-085",
|
|
"wide-086",
|
|
"wide-087",
|
|
"wide-088",
|
|
"wide-089",
|
|
"wide-090",
|
|
"wide-091",
|
|
"wide-092",
|
|
"wide-093",
|
|
"wide-094",
|
|
"wide-095",
|
|
"wide-096",
|
|
"wide-097",
|
|
"wide-098",
|
|
"wide-099",
|
|
"wide-100",
|
|
"wide-101",
|
|
"wide-102",
|
|
"wide-103",
|
|
"wide-104",
|
|
"wide-105",
|
|
"wide-106",
|
|
"wide-107",
|
|
"wide-108",
|
|
"wide-109",
|
|
"wide-110",
|
|
"wide-111",
|
|
"wide-112",
|
|
"wide-113",
|
|
"wide-114",
|
|
"wide-115",
|
|
"wide-116",
|
|
"wide-117",
|
|
"wide-118",
|
|
"wide-119",
|
|
"wide-120",
|
|
"wide-121",
|
|
"wide-122",
|
|
"wide-123",
|
|
"wide-124",
|
|
"wide-125",
|
|
"wide-126",
|
|
"wide-127",
|
|
"wide-128",
|
|
"wide-129",
|
|
"wide-130",
|
|
"wide-131",
|
|
"wide-132",
|
|
"wide-133",
|
|
"wide-134",
|
|
"wide-135",
|
|
"wide-136",
|
|
"wide-137",
|
|
"wide-138",
|
|
"wide-139",
|
|
"wide-140",
|
|
"wide-141",
|
|
"wide-142",
|
|
"wide-143",
|
|
"wide-144",
|
|
"wide-145",
|
|
"wide-146",
|
|
"wide-147",
|
|
"wide-148",
|
|
"wide-149",
|
|
"wide-150",
|
|
"wide-151",
|
|
"wide-152",
|
|
"wide-153",
|
|
"wide-154",
|
|
"wide-155",
|
|
"wide-156",
|
|
"wide-157",
|
|
"wide-158",
|
|
"wide-159",
|
|
"wide-160",
|
|
"wide-161",
|
|
"wide-162",
|
|
"wide-163",
|
|
"wide-164",
|
|
"wide-165",
|
|
"wide-166",
|
|
"wide-167",
|
|
"wide-168",
|
|
"wide-169",
|
|
"wide-170",
|
|
"wide-171",
|
|
"wide-172",
|
|
"wide-173",
|
|
"wide-174",
|
|
"wide-175",
|
|
"wide-176",
|
|
"wide-177",
|
|
"wide-178",
|
|
"wide-179",
|
|
"wide-180",
|
|
"wide-181",
|
|
"wide-182",
|
|
"wide-183",
|
|
"wide-184",
|
|
"wide-185",
|
|
"wide-186",
|
|
"wide-187",
|
|
"wide-188",
|
|
"wide-189",
|
|
"wide-190",
|
|
"wide-191",
|
|
"wide-192",
|
|
"wide-193",
|
|
"wide-194",
|
|
"wide-195",
|
|
"wide-196",
|
|
"wide-197",
|
|
"wide-198",
|
|
"wide-199",
|
|
"wide-200",
|
|
"wide-201",
|
|
"wide-202",
|
|
"wide-203",
|
|
"wide-204",
|
|
"wide-205",
|
|
"wide-206",
|
|
"wide-207",
|
|
"wide-208",
|
|
"wide-209",
|
|
"wide-210",
|
|
"wide-211",
|
|
"wide-212",
|
|
"wide-213",
|
|
"wide-214",
|
|
"wide-215",
|
|
"wide-216",
|
|
"wide-217",
|
|
"wide-218",
|
|
"wide-219",
|
|
"wide-220",
|
|
"wide-221",
|
|
"wide-222",
|
|
"wide-223",
|
|
"wide-224",
|
|
"wide-225",
|
|
"wide-226",
|
|
"wide-227",
|
|
"wide-228",
|
|
"wide-229",
|
|
"wide-230",
|
|
"wide-231",
|
|
"wide-232",
|
|
"wide-233",
|
|
"wide-234",
|
|
"wide-235",
|
|
"wide-236",
|
|
"wide-237",
|
|
"wide-238",
|
|
"wide-239",
|
|
"wide-240",
|
|
"wide-241",
|
|
"wide-242",
|
|
"wide-243",
|
|
"wide-244",
|
|
"wide-245",
|
|
"wide-246",
|
|
"wide-247",
|
|
"wide-248",
|
|
"wide-249",
|
|
"wide-250",
|
|
"wide-251",
|
|
"wide-252",
|
|
"wide-253",
|
|
"wide-254",
|
|
"wide-255",
|
|
"wide-256",
|
|
"wide-257",
|
|
"wide-258",
|
|
"wide-259",
|
|
"wide-260",
|
|
"wide-261",
|
|
"wide-262",
|
|
"wide-263",
|
|
"wide-264",
|
|
"wide-265",
|
|
"wide-266",
|
|
"wide-267",
|
|
"wide-268",
|
|
"wide-269",
|
|
"wide-270",
|
|
"wide-271",
|
|
"wide-272",
|
|
"wide-273",
|
|
"wide-274",
|
|
"wide-275",
|
|
"wide-276",
|
|
"wide-277",
|
|
"wide-278",
|
|
"wide-279",
|
|
"wide-280",
|
|
"wide-281",
|
|
"wide-282",
|
|
"wide-283",
|
|
"wide-284",
|
|
"wide-285",
|
|
"wide-286",
|
|
"wide-287",
|
|
"wide-288",
|
|
"wide-289",
|
|
"wide-290",
|
|
"wide-291",
|
|
"wide-292",
|
|
"wide-293",
|
|
"wide-294",
|
|
"wide-295",
|
|
"wide-296",
|
|
"wide-297",
|
|
"wide-298",
|
|
"wide-299",
|
|
"wide-300",
|
|
"wide-301",
|
|
"wide-302",
|
|
"wide-303",
|
|
"wide-304",
|
|
"wide-305",
|
|
"wide-306",
|
|
"wide-307",
|
|
"wide-308",
|
|
"wide-309",
|
|
"wide-310",
|
|
"wide-311",
|
|
"wide-312",
|
|
"wide-313",
|
|
"wide-314",
|
|
"wide-315",
|
|
"wide-316",
|
|
"wide-317",
|
|
"wide-318",
|
|
"wide-319",
|
|
"wide-320"
|
|
],
|
|
"selected_count": 320,
|
|
"selected_dev_overlap": 42,
|
|
"corrective_request_limit": 1,
|
|
"format_repair_limit": 1,
|
|
"repair_policy": "At most one total corrective request for format, citations or retrieval-context violations. Same question/context, no gold; retain all attempts.",
|
|
"dev_results_sha256": "4af9b52c250c837707c0690d892ec833347b3d3ee71d338ec631f6450eef8045",
|
|
"policy": "Evaluate every declared regression case, including DEV overlap; retain all failures and stop tuning after this freeze.",
|
|
"selected_categories": {
|
|
"shared_keyword_pair": 40,
|
|
"same_academic_year_pair": 40,
|
|
"same_start_year_pair": 32,
|
|
"person_title_year": 24,
|
|
"person_keyword_title": 32,
|
|
"document_author_category_multi_hop": 24,
|
|
"person_multiple_categories": 24,
|
|
"candidate_set_aggregation": 32,
|
|
"conjunctive_disambiguation": 40,
|
|
"negative_lookup": 32
|
|
},
|
|
"limitations": [
|
|
"All questions and previous model outputs are known; this is historical regression.",
|
|
"Harness supplies graph context; native OpenWebUI tool choice and browser UI are outside this experiment.",
|
|
"Checks structured factual claims, their source support/citations and required prose values; not a general semantic judge of arbitrary prose."
|
|
]
|
|
}
|