252 lines
5.1 KiB
JSON
252 lines
5.1 KiB
JSON
{
|
|
"manifest_sha256": "3b2b844562879759d09fec8ad2aeaeafc4bd78201db1200906906a44cea022bf",
|
|
"split": "TEST",
|
|
"complete": true,
|
|
"assessment": "Automated source-backed assessment, not human validation or global accuracy.",
|
|
"rag": {
|
|
"total": 80,
|
|
"passed": 27,
|
|
"success_percent": 33.75,
|
|
"target_met": false,
|
|
"wilson_95_percent": [
|
|
24.35,
|
|
44.64
|
|
],
|
|
"issues": {
|
|
"delivery_rejected": 30,
|
|
"answer_rejected": 30,
|
|
"false_refusal": 31,
|
|
"correct": 19,
|
|
"wrong_phase": 2,
|
|
"judge_unavailable": 4,
|
|
"unsupported_time": 8,
|
|
"appropriate_response": 5,
|
|
"citations": 4,
|
|
"coverage": 7,
|
|
"invented_fact": 2,
|
|
"missing_citation": 2,
|
|
"missing_requirement": 6,
|
|
"readability": 2,
|
|
"unsupported_citation": 1,
|
|
"wrong_status": 2,
|
|
"unsupported_number": 3,
|
|
"wrong_work": 1
|
|
},
|
|
"categories": {
|
|
"absent_work_year": {
|
|
"total": 10,
|
|
"passed": 10
|
|
},
|
|
"assignment_vs_record": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"dated_plans": {
|
|
"total": 10,
|
|
"passed": 7
|
|
},
|
|
"dated_state": {
|
|
"total": 10,
|
|
"passed": 4
|
|
},
|
|
"goals": {
|
|
"total": 10,
|
|
"passed": 2
|
|
},
|
|
"inventory": {
|
|
"total": 10,
|
|
"passed": 4
|
|
},
|
|
"latest_complete_record": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"record_identity": {
|
|
"total": 10,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"retrieval": {
|
|
"candidate_coverage": {
|
|
"all_required": 35,
|
|
"evaluable": 40
|
|
},
|
|
"selected_coverage": {
|
|
"all_required": 13,
|
|
"evaluable": 31
|
|
}
|
|
},
|
|
"dimensions": {
|
|
"correct": {
|
|
"passed": 27,
|
|
"total": 80,
|
|
"judge_evaluable": 46
|
|
},
|
|
"coverage": {
|
|
"passed": 39,
|
|
"total": 80,
|
|
"judge_evaluable": 46
|
|
},
|
|
"citations": {
|
|
"passed": 42,
|
|
"total": 80,
|
|
"judge_evaluable": 46
|
|
},
|
|
"appropriate_response": {
|
|
"passed": 41,
|
|
"total": 80,
|
|
"judge_evaluable": 46
|
|
},
|
|
"clarity": {
|
|
"passed": 44,
|
|
"total": 80
|
|
}
|
|
},
|
|
"false_refusals": 31,
|
|
"delivery_failure_layers": {
|
|
"scope": 10,
|
|
"delivery": 22,
|
|
"verification": 6,
|
|
"fulfilment": 10,
|
|
"answer": 1,
|
|
"evidence": 3,
|
|
"retrieval": 1
|
|
},
|
|
"critical_error_questions": 19,
|
|
"synthesis": {
|
|
"deterministic": 51,
|
|
"fallback": 12,
|
|
"accepted": 17
|
|
},
|
|
"answer_latency_seconds": {
|
|
"median": 22.08,
|
|
"p95": 77.59
|
|
}
|
|
},
|
|
"graphrag": {
|
|
"total": 80,
|
|
"passed": 27,
|
|
"success_percent": 33.75,
|
|
"target_met": false,
|
|
"wilson_95_percent": [
|
|
24.35,
|
|
44.64
|
|
],
|
|
"issues": {
|
|
"delivery_rejected": 30,
|
|
"answer_rejected": 30,
|
|
"false_refusal": 32,
|
|
"correct": 17,
|
|
"unsupported_time": 9,
|
|
"judge_unavailable": 6,
|
|
"citations": 5,
|
|
"wrong_work": 1,
|
|
"wrong_status": 1,
|
|
"unsupported_number": 4,
|
|
"appropriate_response": 3,
|
|
"coverage": 5,
|
|
"invented_fact": 1,
|
|
"missing_citation": 2,
|
|
"missing_requirement": 4,
|
|
"readability": 1,
|
|
"wrong_phase": 1
|
|
},
|
|
"categories": {
|
|
"absent_work_year": {
|
|
"total": 10,
|
|
"passed": 10
|
|
},
|
|
"assignment_vs_record": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"dated_plans": {
|
|
"total": 10,
|
|
"passed": 7
|
|
},
|
|
"dated_state": {
|
|
"total": 10,
|
|
"passed": 4
|
|
},
|
|
"goals": {
|
|
"total": 10,
|
|
"passed": 2
|
|
},
|
|
"inventory": {
|
|
"total": 10,
|
|
"passed": 4
|
|
},
|
|
"latest_complete_record": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"record_identity": {
|
|
"total": 10,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"retrieval": {
|
|
"candidate_coverage": {
|
|
"all_required": 35,
|
|
"evaluable": 40
|
|
},
|
|
"selected_coverage": {
|
|
"all_required": 12,
|
|
"evaluable": 30
|
|
}
|
|
},
|
|
"dimensions": {
|
|
"correct": {
|
|
"passed": 27,
|
|
"total": 80,
|
|
"judge_evaluable": 44
|
|
},
|
|
"coverage": {
|
|
"passed": 39,
|
|
"total": 80,
|
|
"judge_evaluable": 44
|
|
},
|
|
"citations": {
|
|
"passed": 39,
|
|
"total": 80,
|
|
"judge_evaluable": 44
|
|
},
|
|
"appropriate_response": {
|
|
"passed": 41,
|
|
"total": 80,
|
|
"judge_evaluable": 44
|
|
},
|
|
"clarity": {
|
|
"passed": 43,
|
|
"total": 80
|
|
}
|
|
},
|
|
"false_refusals": 32,
|
|
"delivery_failure_layers": {
|
|
"scope": 10,
|
|
"fulfilment": 11,
|
|
"delivery": 22,
|
|
"verification": 6,
|
|
"evidence": 3,
|
|
"retrieval": 1
|
|
},
|
|
"critical_error_questions": 17,
|
|
"synthesis": {
|
|
"deterministic": 52,
|
|
"accepted": 17,
|
|
"fallback": 11
|
|
},
|
|
"answer_latency_seconds": {
|
|
"median": 20.15,
|
|
"p95": 68.72
|
|
}
|
|
},
|
|
"paired": {
|
|
"questions": 80,
|
|
"both_pass": 27,
|
|
"rag_only": 0,
|
|
"graph_only": 0
|
|
},
|
|
"valid_snapshot": true
|
|
}
|