179 lines
5.0 KiB
JSON
179 lines
5.0 KiB
JSON
{
|
|
"completed_at_utc": "2026-10-25T16:30:27.413476+00:00",
|
|
"release": "v5.1",
|
|
"release_tag": "graphrag-v5.1-frozen",
|
|
"evaluation_kind": "historical_regression",
|
|
"graph_dev": {
|
|
"questions": 722,
|
|
"source_checked_count": 722,
|
|
"all_facts_supported_count": 722,
|
|
"strict_count": 722,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.042052,
|
|
"latency_median_seconds": 0.040355,
|
|
"latency_p95_seconds": 0.052196
|
|
},
|
|
"graph_regression": {
|
|
"questions": 320,
|
|
"source_checked_count": 320,
|
|
"all_facts_supported_count": 320,
|
|
"strict_count": 320,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.042475,
|
|
"latency_median_seconds": 0.042369,
|
|
"latency_p95_seconds": 0.049721
|
|
},
|
|
"generation_dev": {
|
|
"questions": 39,
|
|
"correct": 38,
|
|
"accuracy": 0.9743589743589743,
|
|
"errors": 0,
|
|
"first_pass_correct": 38,
|
|
"format_repair_attempted": 0,
|
|
"normalized_first_response": 0
|
|
},
|
|
"generation_regression": {
|
|
"questions": 320,
|
|
"correct": 317,
|
|
"accuracy": 0.990625,
|
|
"errors": 0,
|
|
"first_pass_correct": 317,
|
|
"format_repair_attempted": 0,
|
|
"normalized_first_response": 0
|
|
},
|
|
"generation_regression_by_category": {
|
|
"candidate_set_aggregation": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"conjunctive_disambiguation": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
},
|
|
"document_author_category_multi_hop": {
|
|
"questions": 24,
|
|
"correct": 24,
|
|
"accuracy": 1.0
|
|
},
|
|
"negative_lookup": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_keyword_title": {
|
|
"questions": 32,
|
|
"correct": 30,
|
|
"accuracy": 0.9375
|
|
},
|
|
"person_multiple_categories": {
|
|
"questions": 24,
|
|
"correct": 24,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_title_year": {
|
|
"questions": 24,
|
|
"correct": 23,
|
|
"accuracy": 0.9583333333333334
|
|
},
|
|
"same_academic_year_pair": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
},
|
|
"same_start_year_pair": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"shared_keyword_pair": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
}
|
|
},
|
|
"generation_failure_reasons": {
|
|
"unsupported_claim": 1,
|
|
"answerability": 2
|
|
},
|
|
"comparison_v4_test_300": {
|
|
"questions": 300,
|
|
"v4_correct": 281,
|
|
"v5_1_correct": 297
|
|
},
|
|
"actual_models": {
|
|
"Qwen3.5-122B-A10B-Q4_K_M": 320
|
|
},
|
|
"all_attempt_token_usage": {
|
|
"completion_tokens": 57707,
|
|
"prompt_tokens": 362926,
|
|
"total_tokens": 420633
|
|
},
|
|
"target": 0.92,
|
|
"target_met": true,
|
|
"automated_tests": {
|
|
"passed": 525,
|
|
"skipped": 4,
|
|
"failed": 0
|
|
},
|
|
"development_overlap_in_final_generation": 39,
|
|
"attempt_history": [
|
|
{
|
|
"name": "strict DEV before assignment-title fix",
|
|
"correct": 719,
|
|
"questions": 722
|
|
},
|
|
{
|
|
"name": "v5 graph regression",
|
|
"correct": 319,
|
|
"questions": 320
|
|
},
|
|
{
|
|
"name": "v5 generation DEV",
|
|
"correct": 38,
|
|
"questions": 39
|
|
}
|
|
],
|
|
"point_3_status": {
|
|
"3.1": "Completed for v5.1 historical regression with declared limitations",
|
|
"3.2": "Completed for v5.1 historical regression with declared limitations",
|
|
"3.3": "Completed for v5.1 historical regression with declared limitations",
|
|
"3.4": "Completed for v5.1 historical regression with declared limitations",
|
|
"3.5": "Completed for v5.1 historical regression with declared limitations",
|
|
"3.6": "Completed for v5.1 historical regression with declared limitations"
|
|
},
|
|
"limitations": [
|
|
"Known questions, profiles, templates and corpus. This is not a new independent holdout.",
|
|
"39 development questions occur in the final 320-case generation regression.",
|
|
"Strict criteria and JSON handling changed relative to v4; comparisons are descriptive, not a controlled causal experiment.",
|
|
"Independent rule-based raw-source oracle, not independent human annotation of the full dataset.",
|
|
"Checks typed factual claims, bindings, citations and required prose values; not a semantic judge of every possible sentence.",
|
|
"OpenWebUI API receives locally supplied graph context; native tool routing remains point 4.",
|
|
"Shared-context contract forwarding and further cross-cutting work are deferred as requested.",
|
|
"External model weights are controlled by the provider; repeated requests may differ."
|
|
]
|
|
}
|