{ "schema": 1, "scope": "Final verification of the proposed answer-repair steps; 96 question-held-out cases in both engines.", "decision": "NOT_CLOSED", "target_94_percent_met": false, "complete": true, "valid_snapshot": true, "source_revision": "c3ec9f611e645a2fe14fe66d518eb0924bf9bb40", "regression": { "sqlite_integrity": "ok", "foreign_key_errors": 0, "chunks": 947, "fts_rows": 947, "stored_embedding_segments": 952, "expected_embedding_segments": 952, "segment_keys_match": true, "split_is_lossless": true, "maximum_segment_tokens": 512, "over_512": 0, "maximum_chunk_tokens": 450, "full_regression": { "passed": 1514, "skipped": 90 }, "focused_evaluator_regression": { "passed": 29 }, "live_graph_regression": { "passed": 5, "isolated_graph_scope": true }, "note": "Operational and regression verification; NOT the TEST answer-accuracy score. Host-local Neo4j probe was unavailable; live tests were run successfully in the internal Docker network.", "source_revision_checks": { "local": "c3ec9f611e645a2fe14fe66d518eb0924bf9bb40", "fork_origin_same_as_local": true, "original_upstream_same_as_local": true, "method": "read-only git ls-remote HEAD; no source mutation" } }, "test": { "rag": { "questions": 96, "false_refusals": 42, "false_refusal_case_ids": [ "T001", "T003", "T007", "T009", "T021", "T022", "T023", "T025", "T027", "T028", "T029", "T031", "T033", "T035", "T037", "T039", "T041", "T045", "T047", "T046", "T048", "T050", "T051", "T052", "T061", "T062", "T063", "T067", "T075", "T081", "T082", "T083", "T085", "T084", "T086", "T087", "T088", "T089", "T091", "T093", "T094", "T096" ], "best_possible_percent_if_every_other_answer_were_perfect": 56.25, "raw_strict_assessment_passed": 23, "raw_strict_assessment_percent_NOT_factual_accuracy": 23.96, "invalid_judgments": 32, "retrieval_reference_coverage": { "candidate_coverage": { "all_required": 48, "evaluable": 84 }, "selected_coverage": { "all_required": 15, "evaluable": 33 } }, "dimensions_raw_automated_NOT_human": { "correct": { "passed": 23, "total": 96, "judge_evaluable": 57 }, "coverage": { "passed": 29, "total": 96, "judge_evaluable": 57 }, "citations": { "passed": 28, "total": 96, "judge_evaluable": 57 }, "appropriate_response": { "passed": 30, "total": 96, "judge_evaluable": 57 }, "clarity": { "passed": 33, "total": 96 } }, "answer_latency_seconds": { "median": 4.43, "p95": 28.63 }, "synthesis": { "deterministic": 80, "accepted": 8, "fallback": 8 } }, "graphrag": { "questions": 96, "false_refusals": 50, "false_refusal_case_ids": [ "T001", "T004", "T003", "T005", "T007", "T009", "T011", "T013", "T015", "T017", "T019", "T020", "T021", "T023", "T025", "T027", "T029", "T031", "T033", "T035", "T037", "T039", "T041", "T046", "T045", "T047", "T048", "T050", "T051", "T052", "T062", "T061", "T063", "T067", "T075", "T081", "T082", "T083", "T084", "T085", "T086", "T087", "T088", "T090", "T089", "T091", "T092", "T093", "T094", "T096" ], "best_possible_percent_if_every_other_answer_were_perfect": 47.92, "raw_strict_assessment_passed": 26, "raw_strict_assessment_percent_NOT_factual_accuracy": 27.08, "invalid_judgments": 34, "retrieval_reference_coverage": { "candidate_coverage": { "all_required": 36, "evaluable": 88 }, "selected_coverage": { "all_required": 15, "evaluable": 33 } }, "dimensions_raw_automated_NOT_human": { "correct": { "passed": 26, "total": 96, "judge_evaluable": 59 }, "coverage": { "passed": 31, "total": 96, "judge_evaluable": 59 }, "citations": { "passed": 30, "total": 96, "judge_evaluable": 59 }, "appropriate_response": { "passed": 32, "total": 96, "judge_evaluable": 59 }, "clarity": { "passed": 38, "total": 96 } }, "answer_latency_seconds": { "median": 3.54, "p95": 24.02 }, "synthesis": { "deterministic": 80, "accepted": 7, "fallback": 9 } } }, "critical_substitution_gate": "Unit regressions passed; zero actual critical substitutions on TEST is NOT established by this unreliable model critic.", "assessment_limitations": [ "The strict first-run scores include invalid judgments as failures and contain both judge false rejections and false acceptances. They are NOT a reliable factual-accuracy estimate.", "False refusals are identified independently by expected answerable source-backed cases versus actual no_answer/ambiguous/rejected backend output.", "Candidate/selected metrics measure reference coverage, not retrieval precision. Open-set cases measure anchor/example coverage only.", "AI-authored and AI-reviewed gold, no human validation; new questions on existing documents, not unseen-document generalization.", "Provider model120-fast is a service alias; hidden provider weights/system behavior cannot be frozen by this repository." ], "confirmed_findings_file": "review.json", "next_required_repairs": [ "Separate name mentions and factual filters from presentation/grounding phrases; allow global discovery without an input student.", "Make the common typed plan authoritative in metadata retrieval and rendering; remove contradictory legacy type/title/list interpretations.", "Retrieve content RAG evidence through independent BM25/E5/RRF candidates, then bind selected evidence to source person/work/phase.", "Improve source-facet relevance and preserve all requested finite task/goal items within the actual text budget instead of a hard five-item cap.", "Present proposed goals as proposals, not as missing evidence or completed results; explicitly synthesize requested common results.", "Repair/calibrate the evaluator on realistic positive and negative DEV profiles before making a new independent accuracy claim." ], "test_integrity": "Do not relabel or rerun this consumed TEST as a fresh independent accuracy result. Future fixes can use these failures as regressions, then require a new held-out assessment.", "claude_frontend": "Actual Claude chat not automated here. Public MCP protocol verification is recorded separately; it does not establish frontend wording fidelity.", "running_api_matches_frozen_runtime": { "checked_files": 83, "mismatches": [] }, "public_mcp": { "checks": [ { "tool": "zpwiki_health", "status": "ok", "expected": null, "passed": true, "seconds": 0.1 }, { "tool": "zpwiki_rag", "status": "ok", "expected": "ok", "passed": true, "seconds": 7.3 }, { "tool": "zpwiki_graphrag", "status": "ok", "expected": "ok", "passed": true, "seconds": 5.5 }, { "tool": "zpwiki_graphrag", "status": "no_answer", "expected": "no_answer", "passed": true, "seconds": 2.9 } ], "all_passed": true, "actual_claude_chat_tested": false } }