148 lines
3.7 KiB
JSON
148 lines
3.7 KiB
JSON
{
|
|
"completed_at_utc": "2026-10-25T02:04:11.837608+00:00",
|
|
"graph_dev": {
|
|
"correct": 722,
|
|
"questions": 722,
|
|
"accuracy": 1.0
|
|
},
|
|
"graph_test": {
|
|
"correct": 320,
|
|
"questions": 320,
|
|
"accuracy": 1.0
|
|
},
|
|
"all_graph_context_facts_source_check": {
|
|
"questions": 320,
|
|
"source_consistent": 319,
|
|
"inconsistent_ids": [
|
|
"wide-080"
|
|
]
|
|
},
|
|
"generation_dev": {
|
|
"questions": 20,
|
|
"correct": 19,
|
|
"accuracy": 0.95,
|
|
"errors": 0
|
|
},
|
|
"generation_test": {
|
|
"questions": 300,
|
|
"correct": 281,
|
|
"accuracy": 0.9366666666666666,
|
|
"errors": 7
|
|
},
|
|
"generation_test_by_category": {
|
|
"candidate_set_aggregation": {
|
|
"questions": 30,
|
|
"correct": 27,
|
|
"accuracy": 0.9
|
|
},
|
|
"conjunctive_disambiguation": {
|
|
"questions": 38,
|
|
"correct": 37,
|
|
"accuracy": 0.9736842105263158
|
|
},
|
|
"document_author_category_multi_hop": {
|
|
"questions": 22,
|
|
"correct": 20,
|
|
"accuracy": 0.9090909090909091
|
|
},
|
|
"negative_lookup": {
|
|
"questions": 30,
|
|
"correct": 30,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_keyword_title": {
|
|
"questions": 30,
|
|
"correct": 21,
|
|
"accuracy": 0.7
|
|
},
|
|
"person_multiple_categories": {
|
|
"questions": 22,
|
|
"correct": 21,
|
|
"accuracy": 0.9545454545454546
|
|
},
|
|
"person_title_year": {
|
|
"questions": 22,
|
|
"correct": 21,
|
|
"accuracy": 0.9545454545454546
|
|
},
|
|
"same_academic_year_pair": {
|
|
"questions": 38,
|
|
"correct": 37,
|
|
"accuracy": 0.9736842105263158
|
|
},
|
|
"same_start_year_pair": {
|
|
"questions": 30,
|
|
"correct": 29,
|
|
"accuracy": 0.9666666666666667
|
|
},
|
|
"shared_keyword_pair": {
|
|
"questions": 38,
|
|
"correct": 38,
|
|
"accuracy": 1.0
|
|
}
|
|
},
|
|
"generation_failure_reasons": {
|
|
"unsupported_or_wrong_subject_claim": 3,
|
|
"missing_required_prose_value": 3,
|
|
"invalid_json": 7,
|
|
"binding_missing_or_split": 6
|
|
},
|
|
"generation_latency_seconds": {
|
|
"mean": 7.617359811466643,
|
|
"median": 7.5836237964977045,
|
|
"p95": 11.524189818002924
|
|
},
|
|
"generation_token_usage": {
|
|
"completion_tokens": 52582,
|
|
"prompt_tokens": 243107,
|
|
"total_tokens": 295689
|
|
},
|
|
"target": 0.92,
|
|
"target_met": true,
|
|
"tests": {
|
|
"passed": 448,
|
|
"skipped": 4
|
|
},
|
|
"point_3_status": {
|
|
"1": "complete_for_declared_v4_protocol",
|
|
"2": "complete_for_declared_v4_protocol",
|
|
"3": "complete_for_declared_v4_protocol",
|
|
"4": "complete_for_declared_v4_protocol",
|
|
"5": "complete_for_declared_v4_protocol",
|
|
"6": "complete_for_declared_v4_protocol"
|
|
},
|
|
"attempt_history": [
|
|
{
|
|
"experiment": "graph_v4",
|
|
"dev": 722,
|
|
"test": 320,
|
|
"correct": 320,
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"experiment": "initial_external_diagnostic",
|
|
"questions": 20,
|
|
"correct_raw": 16,
|
|
"role_after_exposure": "dev"
|
|
},
|
|
{
|
|
"experiment": "revised_generation_dev",
|
|
"questions": 20,
|
|
"correct": 19
|
|
},
|
|
{
|
|
"experiment": "generation_v4_test",
|
|
"questions": 300,
|
|
"correct": 281,
|
|
"attempt": 1
|
|
}
|
|
],
|
|
"limitations": [
|
|
"Known corpus, profiles, vocabulary and template families; no independent human annotation of all questions.",
|
|
"Strict generation scoring includes malformed JSON and incomplete prose; not every failure is a factual falsehood.",
|
|
"Retrieval strict checks required proofs, not every extra returned fact. Supplementary all-context source check is reported separately.",
|
|
"OpenWebUI API with harness-supplied context, not native tool routing or browser use.",
|
|
"External server model weights are not locally frozen; raw responses and returned model identity are retained."
|
|
]
|
|
}
|