271 lines
6.9 KiB
JSON
271 lines
6.9 KiB
JSON
{
|
|
"release": "graphrag-v7-frozen",
|
|
"graph_dev": {
|
|
"questions": 1046,
|
|
"source_checked_count": 1046,
|
|
"all_facts_supported_count": 1046,
|
|
"strict_count": 1046,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.081848,
|
|
"latency_median_seconds": 0.071172,
|
|
"latency_p95_seconds": 0.113998
|
|
},
|
|
"graph_new_test": {
|
|
"questions": 80,
|
|
"source_checked_count": 80,
|
|
"all_facts_supported_count": 80,
|
|
"strict_count": 80,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.093012,
|
|
"latency_median_seconds": 0.091794,
|
|
"latency_p95_seconds": 0.127876
|
|
},
|
|
"graph_historical_regression": {
|
|
"questions": 320,
|
|
"source_checked_count": 320,
|
|
"all_facts_supported_count": 320,
|
|
"strict_count": 320,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.081877,
|
|
"latency_median_seconds": 0.078039,
|
|
"latency_p95_seconds": 0.119541
|
|
},
|
|
"model_dev": {
|
|
"questions": 46,
|
|
"correct": 46,
|
|
"accuracy": 1.0,
|
|
"errors": 0,
|
|
"first_pass_correct": 45,
|
|
"format_repair_attempted": 1,
|
|
"corrective_request_count": 1,
|
|
"context_repair_attempted": 0,
|
|
"normalized_first_response": 0
|
|
},
|
|
"model_new_test": {
|
|
"questions": 80,
|
|
"correct": 80,
|
|
"accuracy": 1.0,
|
|
"errors": 0,
|
|
"first_pass_correct": 77,
|
|
"format_repair_attempted": 1,
|
|
"corrective_request_count": 3,
|
|
"context_repair_attempted": 2,
|
|
"normalized_first_response": 0
|
|
},
|
|
"model_historical_regression": {
|
|
"questions": 320,
|
|
"correct": 319,
|
|
"accuracy": 0.996875,
|
|
"errors": 0,
|
|
"first_pass_correct": 310,
|
|
"format_repair_attempted": 3,
|
|
"corrective_request_count": 10,
|
|
"context_repair_attempted": 7,
|
|
"normalized_first_response": 0
|
|
},
|
|
"model_new_test_by_category": {
|
|
"candidate_set_aggregation": {
|
|
"questions": 7,
|
|
"correct": 7,
|
|
"accuracy": 1.0
|
|
},
|
|
"conjunctive_disambiguation": {
|
|
"questions": 6,
|
|
"correct": 6,
|
|
"accuracy": 1.0
|
|
},
|
|
"document_author_category_multi_hop": {
|
|
"questions": 2,
|
|
"correct": 2,
|
|
"accuracy": 1.0
|
|
},
|
|
"negative_lookup": {
|
|
"questions": 8,
|
|
"correct": 8,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_keyword_title": {
|
|
"questions": 10,
|
|
"correct": 10,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_multiple_categories": {
|
|
"questions": 7,
|
|
"correct": 7,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_title_year": {
|
|
"questions": 10,
|
|
"correct": 10,
|
|
"accuracy": 1.0
|
|
},
|
|
"same_academic_year_pair": {
|
|
"questions": 10,
|
|
"correct": 10,
|
|
"accuracy": 1.0
|
|
},
|
|
"same_start_year_pair": {
|
|
"questions": 10,
|
|
"correct": 10,
|
|
"accuracy": 1.0
|
|
},
|
|
"shared_keyword_pair": {
|
|
"questions": 10,
|
|
"correct": 10,
|
|
"accuracy": 1.0
|
|
}
|
|
},
|
|
"model_regression_by_category": {
|
|
"candidate_set_aggregation": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"conjunctive_disambiguation": {
|
|
"questions": 40,
|
|
"correct": 39,
|
|
"accuracy": 0.975
|
|
},
|
|
"document_author_category_multi_hop": {
|
|
"questions": 24,
|
|
"correct": 24,
|
|
"accuracy": 1.0
|
|
},
|
|
"negative_lookup": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_keyword_title": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_multiple_categories": {
|
|
"questions": 24,
|
|
"correct": 24,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_title_year": {
|
|
"questions": 24,
|
|
"correct": 24,
|
|
"accuracy": 1.0
|
|
},
|
|
"same_academic_year_pair": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
},
|
|
"same_start_year_pair": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"shared_keyword_pair": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
}
|
|
},
|
|
"protocol": {
|
|
"seed": 20261026,
|
|
"dev_count": 1046,
|
|
"fresh_test_count": 80,
|
|
"test_categories": {
|
|
"shared_keyword_pair": 10,
|
|
"same_academic_year_pair": 10,
|
|
"same_start_year_pair": 10,
|
|
"person_title_year": 10,
|
|
"person_keyword_title": 10,
|
|
"document_author_category_multi_hop": 2,
|
|
"person_multiple_categories": 7,
|
|
"candidate_set_aggregation": 7,
|
|
"conjunctive_disambiguation": 6,
|
|
"negative_lookup": 8
|
|
},
|
|
"test_positives": 72,
|
|
"test_negatives": 8,
|
|
"available_groups": {
|
|
"shared_keyword_pair": 1883,
|
|
"same_academic_year_pair": 535,
|
|
"same_start_year_pair": 153,
|
|
"person_title_year": 30,
|
|
"person_keyword_title": 146,
|
|
"document_author_category_multi_hop": 2,
|
|
"person_multiple_categories": 7,
|
|
"candidate_set_aggregation": 54,
|
|
"conjunctive_disambiguation": 628,
|
|
"negative_lookup": 106
|
|
},
|
|
"old_gold_corrections": 10,
|
|
"regression_count": 320,
|
|
"llm_dev_count": 46,
|
|
"target": 0.92,
|
|
"policy": "Freeze after complete DEV; run fresh TEST once, retain every failure. Regression remains separate.",
|
|
"annotation": "Independent raw-source oracle plus AI source review; not human review."
|
|
},
|
|
"tests": {
|
|
"passed": 591,
|
|
"skipped": 0
|
|
},
|
|
"source_review": {
|
|
"sample": 20,
|
|
"supported": 20,
|
|
"reviewer": "AI direct raw Markdown review, not human"
|
|
},
|
|
"functional_scenarios": {
|
|
"passed": 9,
|
|
"total": 9
|
|
},
|
|
"historical_rag_replay": {
|
|
"passed": 5,
|
|
"total": 5,
|
|
"new_generation": false
|
|
},
|
|
"failed_test_ids": [],
|
|
"failed_regression_ids": [
|
|
"wide-285"
|
|
],
|
|
"limitations": [
|
|
"Known corpus, persons and template families; new query groups, not unseen profiles.",
|
|
"Source oracle plus AI review; no human annotation.",
|
|
"Only two fresh document-author/category groups and seven fresh category-list groups remain.",
|
|
"End-to-end native OpenWebUI tool routing is point 4.",
|
|
"Final prose is deterministic rendering of verified model-selected claims; this is not free-prose generation accuracy.",
|
|
"Gold corrections and changed delivery protocol prevent a controlled accuracy comparison with v6."
|
|
]
|
|
}
|