302 lines
8.4 KiB
JSON
302 lines
8.4 KiB
JSON
{
|
|
"schema": 1,
|
|
"scope": "Final verification of the proposed answer-repair steps; 96 question-held-out cases in both engines.",
|
|
"decision": "NOT_CLOSED",
|
|
"target_94_percent_met": false,
|
|
"complete": true,
|
|
"valid_snapshot": true,
|
|
"source_revision": "c3ec9f611e645a2fe14fe66d518eb0924bf9bb40",
|
|
"regression": {
|
|
"sqlite_integrity": "ok",
|
|
"foreign_key_errors": 0,
|
|
"chunks": 947,
|
|
"fts_rows": 947,
|
|
"stored_embedding_segments": 952,
|
|
"expected_embedding_segments": 952,
|
|
"segment_keys_match": true,
|
|
"split_is_lossless": true,
|
|
"maximum_segment_tokens": 512,
|
|
"over_512": 0,
|
|
"maximum_chunk_tokens": 450,
|
|
"full_regression": {
|
|
"passed": 1514,
|
|
"skipped": 90
|
|
},
|
|
"focused_evaluator_regression": {
|
|
"passed": 29
|
|
},
|
|
"live_graph_regression": {
|
|
"passed": 5,
|
|
"isolated_graph_scope": true
|
|
},
|
|
"note": "Operational and regression verification; NOT the TEST answer-accuracy score. Host-local Neo4j probe was unavailable; live tests were run successfully in the internal Docker network.",
|
|
"source_revision_checks": {
|
|
"local": "c3ec9f611e645a2fe14fe66d518eb0924bf9bb40",
|
|
"fork_origin_same_as_local": true,
|
|
"original_upstream_same_as_local": true,
|
|
"method": "read-only git ls-remote HEAD; no source mutation"
|
|
}
|
|
},
|
|
"test": {
|
|
"rag": {
|
|
"questions": 96,
|
|
"false_refusals": 42,
|
|
"false_refusal_case_ids": [
|
|
"T001",
|
|
"T003",
|
|
"T007",
|
|
"T009",
|
|
"T021",
|
|
"T022",
|
|
"T023",
|
|
"T025",
|
|
"T027",
|
|
"T028",
|
|
"T029",
|
|
"T031",
|
|
"T033",
|
|
"T035",
|
|
"T037",
|
|
"T039",
|
|
"T041",
|
|
"T045",
|
|
"T047",
|
|
"T046",
|
|
"T048",
|
|
"T050",
|
|
"T051",
|
|
"T052",
|
|
"T061",
|
|
"T062",
|
|
"T063",
|
|
"T067",
|
|
"T075",
|
|
"T081",
|
|
"T082",
|
|
"T083",
|
|
"T085",
|
|
"T084",
|
|
"T086",
|
|
"T087",
|
|
"T088",
|
|
"T089",
|
|
"T091",
|
|
"T093",
|
|
"T094",
|
|
"T096"
|
|
],
|
|
"best_possible_percent_if_every_other_answer_were_perfect": 56.25,
|
|
"raw_strict_assessment_passed": 23,
|
|
"raw_strict_assessment_percent_NOT_factual_accuracy": 23.96,
|
|
"invalid_judgments": 32,
|
|
"retrieval_reference_coverage": {
|
|
"candidate_coverage": {
|
|
"all_required": 48,
|
|
"evaluable": 84
|
|
},
|
|
"selected_coverage": {
|
|
"all_required": 15,
|
|
"evaluable": 33
|
|
}
|
|
},
|
|
"dimensions_raw_automated_NOT_human": {
|
|
"correct": {
|
|
"passed": 23,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"coverage": {
|
|
"passed": 29,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"citations": {
|
|
"passed": 28,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"appropriate_response": {
|
|
"passed": 30,
|
|
"total": 96,
|
|
"judge_evaluable": 57
|
|
},
|
|
"clarity": {
|
|
"passed": 33,
|
|
"total": 96
|
|
}
|
|
},
|
|
"answer_latency_seconds": {
|
|
"median": 4.43,
|
|
"p95": 28.63
|
|
},
|
|
"synthesis": {
|
|
"deterministic": 80,
|
|
"accepted": 8,
|
|
"fallback": 8
|
|
}
|
|
},
|
|
"graphrag": {
|
|
"questions": 96,
|
|
"false_refusals": 50,
|
|
"false_refusal_case_ids": [
|
|
"T001",
|
|
"T004",
|
|
"T003",
|
|
"T005",
|
|
"T007",
|
|
"T009",
|
|
"T011",
|
|
"T013",
|
|
"T015",
|
|
"T017",
|
|
"T019",
|
|
"T020",
|
|
"T021",
|
|
"T023",
|
|
"T025",
|
|
"T027",
|
|
"T029",
|
|
"T031",
|
|
"T033",
|
|
"T035",
|
|
"T037",
|
|
"T039",
|
|
"T041",
|
|
"T046",
|
|
"T045",
|
|
"T047",
|
|
"T048",
|
|
"T050",
|
|
"T051",
|
|
"T052",
|
|
"T062",
|
|
"T061",
|
|
"T063",
|
|
"T067",
|
|
"T075",
|
|
"T081",
|
|
"T082",
|
|
"T083",
|
|
"T084",
|
|
"T085",
|
|
"T086",
|
|
"T087",
|
|
"T088",
|
|
"T090",
|
|
"T089",
|
|
"T091",
|
|
"T092",
|
|
"T093",
|
|
"T094",
|
|
"T096"
|
|
],
|
|
"best_possible_percent_if_every_other_answer_were_perfect": 47.92,
|
|
"raw_strict_assessment_passed": 26,
|
|
"raw_strict_assessment_percent_NOT_factual_accuracy": 27.08,
|
|
"invalid_judgments": 34,
|
|
"retrieval_reference_coverage": {
|
|
"candidate_coverage": {
|
|
"all_required": 36,
|
|
"evaluable": 88
|
|
},
|
|
"selected_coverage": {
|
|
"all_required": 15,
|
|
"evaluable": 33
|
|
}
|
|
},
|
|
"dimensions_raw_automated_NOT_human": {
|
|
"correct": {
|
|
"passed": 26,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"coverage": {
|
|
"passed": 31,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"citations": {
|
|
"passed": 30,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"appropriate_response": {
|
|
"passed": 32,
|
|
"total": 96,
|
|
"judge_evaluable": 59
|
|
},
|
|
"clarity": {
|
|
"passed": 38,
|
|
"total": 96
|
|
}
|
|
},
|
|
"answer_latency_seconds": {
|
|
"median": 3.54,
|
|
"p95": 24.02
|
|
},
|
|
"synthesis": {
|
|
"deterministic": 80,
|
|
"accepted": 7,
|
|
"fallback": 9
|
|
}
|
|
}
|
|
},
|
|
"critical_substitution_gate": "Unit regressions passed; zero actual critical substitutions on TEST is NOT established by this unreliable model critic.",
|
|
"assessment_limitations": [
|
|
"The strict first-run scores include invalid judgments as failures and contain both judge false rejections and false acceptances. They are NOT a reliable factual-accuracy estimate.",
|
|
"False refusals are identified independently by expected answerable source-backed cases versus actual no_answer/ambiguous/rejected backend output.",
|
|
"Candidate/selected metrics measure reference coverage, not retrieval precision. Open-set cases measure anchor/example coverage only.",
|
|
"AI-authored and AI-reviewed gold, no human validation; new questions on existing documents, not unseen-document generalization.",
|
|
"Provider model120-fast is a service alias; hidden provider weights/system behavior cannot be frozen by this repository."
|
|
],
|
|
"confirmed_findings_file": "review.json",
|
|
"next_required_repairs": [
|
|
"Separate name mentions and factual filters from presentation/grounding phrases; allow global discovery without an input student.",
|
|
"Make the common typed plan authoritative in metadata retrieval and rendering; remove contradictory legacy type/title/list interpretations.",
|
|
"Retrieve content RAG evidence through independent BM25/E5/RRF candidates, then bind selected evidence to source person/work/phase.",
|
|
"Improve source-facet relevance and preserve all requested finite task/goal items within the actual text budget instead of a hard five-item cap.",
|
|
"Present proposed goals as proposals, not as missing evidence or completed results; explicitly synthesize requested common results.",
|
|
"Repair/calibrate the evaluator on realistic positive and negative DEV profiles before making a new independent accuracy claim."
|
|
],
|
|
"test_integrity": "Do not relabel or rerun this consumed TEST as a fresh independent accuracy result. Future fixes can use these failures as regressions, then require a new held-out assessment.",
|
|
"claude_frontend": "Actual Claude chat not automated here. Public MCP protocol verification is recorded separately; it does not establish frontend wording fidelity.",
|
|
"running_api_matches_frozen_runtime": {
|
|
"checked_files": 83,
|
|
"mismatches": []
|
|
},
|
|
"public_mcp": {
|
|
"checks": [
|
|
{
|
|
"tool": "zpwiki_health",
|
|
"status": "ok",
|
|
"expected": null,
|
|
"passed": true,
|
|
"seconds": 0.1
|
|
},
|
|
{
|
|
"tool": "zpwiki_rag",
|
|
"status": "ok",
|
|
"expected": "ok",
|
|
"passed": true,
|
|
"seconds": 7.3
|
|
},
|
|
{
|
|
"tool": "zpwiki_graphrag",
|
|
"status": "ok",
|
|
"expected": "ok",
|
|
"passed": true,
|
|
"seconds": 5.5
|
|
},
|
|
{
|
|
"tool": "zpwiki_graphrag",
|
|
"status": "no_answer",
|
|
"expected": "no_answer",
|
|
"passed": true,
|
|
"seconds": 2.9
|
|
}
|
|
],
|
|
"all_passed": true,
|
|
"actual_claude_chat_tested": false
|
|
}
|
|
}
|