{
  "purpose": "Paired question-held-out evaluation of the current ZP Agent, not a global accuracy claim.",
  "source": "https://arxiv.org/abs/2309.15217",
  "method": "RAGAS-inspired separation of retrieval, grounded final answers and readability; not the official RAGAS implementation or scores.",
  "test_cases": 96,
  "engines": [
    "rag",
    "graphrag"
  ],
  "same_model": "model120-fast",
  "same_question_and_limit": true,
  "threshold": {
    "per_engine_minimum": 91,
    "denominator": 96,
    "requested_fraction": 0.94,
    "critical_substitutions_allowed": 0
  },
  "strict_success": [
    "accepted delivery",
    "expected response behavior",
    "all facts supported",
    "all required conditions covered",
    "source citations supported",
    "readability >=3 of4",
    "no critical person/work/phase/event/date/number error"
  ],
  "retrieval": [
    "Candidate-reference coverage before selection",
    "Selected-reference coverage after selection",
    "Content RAG uses the content checkpoint; metadata RAG uses BM25/E5/RRF",
    "Content GraphRAG uses bounded local Neo4j paths; metadata uses parametrized graph queries",
    "Open-set anchor/example coverage is not exhaustive peer/discovery recall"
  ],
  "independence": [
    "DEV questions and audited paraphrases are separate",
    "TEST questions authored against raw Markdown before backend execution",
    "No exact overlap with scanned historical question datasets",
    "No TEST-directed production changes or gold changes after freeze"
  ],
  "assessment_limits": [
    "AI source-check and model judge, no human review claimed",
    "Same model in answer pipeline and judge can make correlated errors",
    "Calibration passed 8/8 synthetic cases, but DEV judge still produces invalid judgments; uncertainty must be reported",
    "Unavailable or invalid judgments remain failures for the conservative strict score; not removed from denominator",
    "Shared public corpus and repeated profiles do not establish unseen-document generalization",
    "Actual Claude chat verification is a manual endpoint check, separate from MCP protocol checks"
  ],
  "artifacts": [
    "test.json",
    "test_manifest.json",
    "test_results.jsonl",
    "test_summary.json",
    "review.json"
  ],
  "review_policy": "Retain first-run model verdicts. Supplemental AI source review may identify evaluator mistakes but must not overwrite the strict result or claim human validation."
}