387 lines
7.5 KiB
JSON
387 lines
7.5 KiB
JSON
{
|
|
"manifest_sha256": "34ac773a62a7744a5cb4576ca14dd3116f65496ade5436e658044bf47dfb5c93",
|
|
"split": "TEST",
|
|
"complete": true,
|
|
"assessment": "Automated source-backed assessment, not human validation or global accuracy.",
|
|
"rag": {
|
|
"total": 96,
|
|
"passed": 23,
|
|
"success_percent": 23.96,
|
|
"target_met": false,
|
|
"wilson_95_percent": [
|
|
16.53,
|
|
33.39
|
|
],
|
|
"issues": {
|
|
"delivery_rejected": 7,
|
|
"answer_rejected": 7,
|
|
"false_refusal": 42,
|
|
"correct": 33,
|
|
"coverage": 28,
|
|
"missing_requirement": 20,
|
|
"unsupported_time": 7,
|
|
"wrong_work": 2,
|
|
"judge_unavailable": 32,
|
|
"citations": 29,
|
|
"unsupported_number": 3,
|
|
"readability": 24,
|
|
"appropriate_response": 26,
|
|
"invented_fact": 14,
|
|
"wrong_person": 4,
|
|
"unsupported_citation": 2,
|
|
"wrong_response_status": 1
|
|
},
|
|
"categories": {
|
|
"advisor_roles": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"ambiguous_person": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"comparison": {
|
|
"total": 8,
|
|
"passed": 0
|
|
},
|
|
"discovery_open_set": {
|
|
"total": 4,
|
|
"passed": 0
|
|
},
|
|
"goals": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"latest_record": {
|
|
"total": 8,
|
|
"passed": 5
|
|
},
|
|
"missing_year": {
|
|
"total": 2,
|
|
"passed": 2
|
|
},
|
|
"name_typo": {
|
|
"total": 2,
|
|
"passed": 0
|
|
},
|
|
"no_shared_year": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"planned_task": {
|
|
"total": 8,
|
|
"passed": 3
|
|
},
|
|
"record_count": {
|
|
"total": 2,
|
|
"passed": 0
|
|
},
|
|
"record_period": {
|
|
"total": 4,
|
|
"passed": 0
|
|
},
|
|
"recorded_metódy": {
|
|
"total": 5,
|
|
"passed": 1
|
|
},
|
|
"recorded_pokrok": {
|
|
"total": 3,
|
|
"passed": 0
|
|
},
|
|
"recorded_problémy": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"recorded_stav": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"recorded_výsledky": {
|
|
"total": 2,
|
|
"passed": 0
|
|
},
|
|
"shared_tags": {
|
|
"total": 3,
|
|
"passed": 3
|
|
},
|
|
"shared_years": {
|
|
"total": 2,
|
|
"passed": 2
|
|
},
|
|
"similarity_open_set": {
|
|
"total": 4,
|
|
"passed": 0
|
|
},
|
|
"study_status": {
|
|
"total": 1,
|
|
"passed": 1
|
|
},
|
|
"supervisor_role": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"tag_work_year_join": {
|
|
"total": 2,
|
|
"passed": 0
|
|
},
|
|
"time_roles": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"title_year": {
|
|
"total": 10,
|
|
"passed": 3
|
|
},
|
|
"unknown_person": {
|
|
"total": 1,
|
|
"passed": 1
|
|
},
|
|
"work_inventory": {
|
|
"total": 8,
|
|
"passed": 2
|
|
}
|
|
},
|
|
"retrieval": {
|
|
"candidate_coverage": {
|
|
"all_required": 48,
|
|
"evaluable": 84
|
|
},
|
|
"selected_coverage": {
|
|
"all_required": 15,
|
|
"evaluable": 33
|
|
}
|
|
},
|
|
"dimensions": {
|
|
"correct": {
|
|
"passed": 23,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"coverage": {
|
|
"passed": 29,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"citations": {
|
|
"passed": 28,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"appropriate_response": {
|
|
"passed": 30,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"clarity": {
|
|
"passed": 33,
|
|
"total": 96
|
|
}
|
|
},
|
|
"false_refusals": 42,
|
|
"critical_error_questions": 33,
|
|
"synthesis": {
|
|
"deterministic": 80,
|
|
"accepted": 8,
|
|
"fallback": 8
|
|
},
|
|
"answer_latency_seconds": {
|
|
"median": 4.43,
|
|
"p95": 28.63
|
|
}
|
|
},
|
|
"graphrag": {
|
|
"total": 96,
|
|
"passed": 26,
|
|
"success_percent": 27.08,
|
|
"target_met": false,
|
|
"wilson_95_percent": [
|
|
19.2,
|
|
36.73
|
|
],
|
|
"issues": {
|
|
"false_refusal": 50,
|
|
"judge_unavailable": 34,
|
|
"correct": 33,
|
|
"unsupported_time": 5,
|
|
"appropriate_response": 27,
|
|
"citations": 29,
|
|
"coverage": 28,
|
|
"invented_fact": 16,
|
|
"wrong_status": 1,
|
|
"readability": 21,
|
|
"wrong_person": 2,
|
|
"missing_requirement": 18,
|
|
"missing_citation": 2,
|
|
"wrong_work": 1,
|
|
"unsupported_number": 4,
|
|
"unsupported_citation": 1,
|
|
"delivery_rejected": 3,
|
|
"answer_rejected": 3
|
|
},
|
|
"categories": {
|
|
"advisor_roles": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"ambiguous_person": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"comparison": {
|
|
"total": 8,
|
|
"passed": 0
|
|
},
|
|
"discovery_open_set": {
|
|
"total": 4,
|
|
"passed": 0
|
|
},
|
|
"goals": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"latest_record": {
|
|
"total": 8,
|
|
"passed": 5
|
|
},
|
|
"missing_year": {
|
|
"total": 2,
|
|
"passed": 2
|
|
},
|
|
"name_typo": {
|
|
"total": 2,
|
|
"passed": 1
|
|
},
|
|
"no_shared_year": {
|
|
"total": 1,
|
|
"passed": 1
|
|
},
|
|
"planned_task": {
|
|
"total": 8,
|
|
"passed": 3
|
|
},
|
|
"record_count": {
|
|
"total": 2,
|
|
"passed": 0
|
|
},
|
|
"record_period": {
|
|
"total": 4,
|
|
"passed": 0
|
|
},
|
|
"recorded_metódy": {
|
|
"total": 5,
|
|
"passed": 1
|
|
},
|
|
"recorded_pokrok": {
|
|
"total": 3,
|
|
"passed": 0
|
|
},
|
|
"recorded_problémy": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"recorded_stav": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"recorded_výsledky": {
|
|
"total": 2,
|
|
"passed": 0
|
|
},
|
|
"shared_tags": {
|
|
"total": 3,
|
|
"passed": 3
|
|
},
|
|
"shared_years": {
|
|
"total": 2,
|
|
"passed": 2
|
|
},
|
|
"similarity_open_set": {
|
|
"total": 4,
|
|
"passed": 0
|
|
},
|
|
"study_status": {
|
|
"total": 1,
|
|
"passed": 1
|
|
},
|
|
"supervisor_role": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"tag_work_year_join": {
|
|
"total": 2,
|
|
"passed": 2
|
|
},
|
|
"time_roles": {
|
|
"total": 1,
|
|
"passed": 0
|
|
},
|
|
"title_year": {
|
|
"total": 10,
|
|
"passed": 0
|
|
},
|
|
"unknown_person": {
|
|
"total": 1,
|
|
"passed": 1
|
|
},
|
|
"work_inventory": {
|
|
"total": 8,
|
|
"passed": 4
|
|
}
|
|
},
|
|
"retrieval": {
|
|
"candidate_coverage": {
|
|
"all_required": 36,
|
|
"evaluable": 88
|
|
},
|
|
"selected_coverage": {
|
|
"all_required": 15,
|
|
"evaluable": 33
|
|
}
|
|
},
|
|
"dimensions": {
|
|
"correct": {
|
|
"passed": 26,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"coverage": {
|
|
"passed": 31,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"citations": {
|
|
"passed": 30,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"appropriate_response": {
|
|
"passed": 32,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"clarity": {
|
|
"passed": 38,
|
|
"total": 96
|
|
}
|
|
},
|
|
"false_refusals": 50,
|
|
"critical_error_questions": 33,
|
|
"synthesis": {
|
|
"deterministic": 80,
|
|
"accepted": 7,
|
|
"fallback": 9
|
|
},
|
|
"answer_latency_seconds": {
|
|
"median": 3.54,
|
|
"p95": 24.02
|
|
}
|
|
},
|
|
"paired": {
|
|
"questions": 96,
|
|
"both_pass": 19,
|
|
"rag_only": 4,
|
|
"graph_only": 7
|
|
},
|
|
"valid_snapshot": true
|
|
}
|