189 lines
5.0 KiB
JSON
189 lines
5.0 KiB
JSON
{
|
|
"evaluation_kind": "historical_regression",
|
|
"model": "model120-fast",
|
|
"graph_dev": {
|
|
"questions": 722,
|
|
"source_checked_count": 722,
|
|
"all_facts_supported_count": 722,
|
|
"strict_count": 722,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.051461,
|
|
"latency_median_seconds": 0.04753,
|
|
"latency_p95_seconds": 0.079209
|
|
},
|
|
"graph_regression": {
|
|
"questions": 320,
|
|
"source_checked_count": 320,
|
|
"all_facts_supported_count": 320,
|
|
"strict_count": 320,
|
|
"strict_accuracy": 1.0,
|
|
"error_count": 0,
|
|
"value_score": 1.0,
|
|
"entity_score": 1.0,
|
|
"relation_score": 1.0,
|
|
"property_score": 1.0,
|
|
"relation_choice_score": 1.0,
|
|
"document_recall": 1.0,
|
|
"source_recall": 1.0,
|
|
"binding_score": 1.0,
|
|
"path_score": 1.0,
|
|
"answerability_accuracy": 1.0,
|
|
"latency_mean_seconds": 0.053195,
|
|
"latency_median_seconds": 0.049496,
|
|
"latency_p95_seconds": 0.075778
|
|
},
|
|
"llm_dev": {
|
|
"questions": 42,
|
|
"correct": 40,
|
|
"accuracy": 0.9523809523809523,
|
|
"errors": 2,
|
|
"first_pass_correct": 40,
|
|
"format_repair_attempted": 2,
|
|
"corrective_request_count": 2,
|
|
"context_repair_attempted": 0,
|
|
"normalized_first_response": 0
|
|
},
|
|
"llm_regression": {
|
|
"questions": 320,
|
|
"correct": 316,
|
|
"accuracy": 0.9875,
|
|
"errors": 3,
|
|
"first_pass_correct": 316,
|
|
"format_repair_attempted": 3,
|
|
"corrective_request_count": 3,
|
|
"context_repair_attempted": 0,
|
|
"normalized_first_response": 0
|
|
},
|
|
"llm_by_category": {
|
|
"candidate_set_aggregation": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"conjunctive_disambiguation": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
},
|
|
"document_author_category_multi_hop": {
|
|
"questions": 24,
|
|
"correct": 23,
|
|
"accuracy": 0.9583333333333334
|
|
},
|
|
"negative_lookup": {
|
|
"questions": 32,
|
|
"correct": 32,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_keyword_title": {
|
|
"questions": 32,
|
|
"correct": 31,
|
|
"accuracy": 0.96875
|
|
},
|
|
"person_multiple_categories": {
|
|
"questions": 24,
|
|
"correct": 24,
|
|
"accuracy": 1.0
|
|
},
|
|
"person_title_year": {
|
|
"questions": 24,
|
|
"correct": 23,
|
|
"accuracy": 0.9583333333333334
|
|
},
|
|
"same_academic_year_pair": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
},
|
|
"same_start_year_pair": {
|
|
"questions": 32,
|
|
"correct": 31,
|
|
"accuracy": 0.96875
|
|
},
|
|
"shared_keyword_pair": {
|
|
"questions": 40,
|
|
"correct": 40,
|
|
"accuracy": 1.0
|
|
}
|
|
},
|
|
"valid_json_responses": 317,
|
|
"source_supported_structured_claims": 317,
|
|
"tests": {
|
|
"passed": 565,
|
|
"skipped": 0,
|
|
"warning": "Existing Starlette/httpx deprecation warning."
|
|
},
|
|
"synthetic_functional_checks": {
|
|
"passed": 9,
|
|
"total": 9
|
|
},
|
|
"rag_live_smoke": {
|
|
"total": 5,
|
|
"factual_and_citation_passes": 5,
|
|
"raw_delivery_accepted": 0,
|
|
"missing_notice": 5,
|
|
"deterministic_delivery_replay_passes": 5,
|
|
"model_accuracy_estimate": false
|
|
},
|
|
"source_review": {
|
|
"total": 34,
|
|
"supported": 31,
|
|
"corrected": 2,
|
|
"conditional_directory_cohort": 1,
|
|
"human_review": "not performed; worksheet available"
|
|
},
|
|
"gold_corrections": 15,
|
|
"regression_positive": 284,
|
|
"regression_negative": 36,
|
|
"llm_failure_details": [
|
|
{
|
|
"id": "wide-104",
|
|
"error": null,
|
|
"missing_text_values": [
|
|
"Martin NaN1"
|
|
],
|
|
"source_supported_claims": true
|
|
},
|
|
{
|
|
"id": "wide-119",
|
|
"error": "InvalidAnswerFormat: two invalid JSON/schema responses",
|
|
"missing_text_values": null,
|
|
"source_supported_claims": false
|
|
},
|
|
{
|
|
"id": "wide-143",
|
|
"error": "InvalidAnswerFormat: two invalid JSON/schema responses",
|
|
"missing_text_values": null,
|
|
"source_supported_claims": false
|
|
},
|
|
{
|
|
"id": "wide-176",
|
|
"error": "InvalidAnswerFormat: two invalid JSON/schema responses",
|
|
"missing_text_values": null,
|
|
"source_supported_claims": false
|
|
}
|
|
],
|
|
"target_accuracy": 0.92,
|
|
"target_met": true,
|
|
"limitations": [
|
|
"Known questions and overlapping DEV; not independent TEST.",
|
|
"15 source-gold corrections; accuracy change vs v5.1 is not controlled comparison.",
|
|
"Direct source review by AI, not a human. One study-start expectation uses directory-cohort fallback.",
|
|
"Strict lexical name check rejects a grammatical inflection; structured claims remain supported.",
|
|
"Graph contexts supplied by evaluation client; native OpenWebUI tool routing belongs to point 4.",
|
|
"URL allowlist is stricter than factual truth of URLs mentioned inside source documents.",
|
|
"Rules check structured claims and selected prose requirements, not every possible natural-language assertion."
|
|
]
|
|
}
|