57 lines
2.4 KiB
JSON
57 lines
2.4 KiB
JSON
{
|
|
"purpose": "Paired question-held-out evaluation of the current ZP Agent, not a global accuracy claim.",
|
|
"source": "https://arxiv.org/abs/2309.15217",
|
|
"method": "RAGAS-inspired separation of retrieval, grounded final answers and readability; not the official RAGAS implementation or scores.",
|
|
"test_cases": 96,
|
|
"engines": [
|
|
"rag",
|
|
"graphrag"
|
|
],
|
|
"same_model": "model120-fast",
|
|
"same_question_and_limit": true,
|
|
"threshold": {
|
|
"per_engine_minimum": 91,
|
|
"denominator": 96,
|
|
"requested_fraction": 0.94,
|
|
"critical_substitutions_allowed": 0
|
|
},
|
|
"strict_success": [
|
|
"accepted delivery",
|
|
"expected response behavior",
|
|
"all facts supported",
|
|
"all required conditions covered",
|
|
"source citations supported",
|
|
"readability >=3 of4",
|
|
"no critical person/work/phase/event/date/number error"
|
|
],
|
|
"retrieval": [
|
|
"Candidate-reference coverage before selection",
|
|
"Selected-reference coverage after selection",
|
|
"Content RAG uses the content checkpoint; metadata RAG uses BM25/E5/RRF",
|
|
"Content GraphRAG uses bounded local Neo4j paths; metadata uses parametrized graph queries",
|
|
"Open-set anchor/example coverage is not exhaustive peer/discovery recall"
|
|
],
|
|
"independence": [
|
|
"DEV questions and audited paraphrases are separate",
|
|
"TEST questions authored against raw Markdown before backend execution",
|
|
"No exact overlap with scanned historical question datasets",
|
|
"No TEST-directed production changes or gold changes after freeze"
|
|
],
|
|
"assessment_limits": [
|
|
"AI source-check and model judge, no human review claimed",
|
|
"Same model in answer pipeline and judge can make correlated errors",
|
|
"Calibration passed 8/8 synthetic cases, but DEV judge still produces invalid judgments; uncertainty must be reported",
|
|
"Unavailable or invalid judgments remain failures for the conservative strict score; not removed from denominator",
|
|
"Shared public corpus and repeated profiles do not establish unseen-document generalization",
|
|
"Actual Claude chat verification is a manual endpoint check, separate from MCP protocol checks"
|
|
],
|
|
"artifacts": [
|
|
"test.json",
|
|
"test_manifest.json",
|
|
"test_results.jsonl",
|
|
"test_summary.json",
|
|
"review.json"
|
|
],
|
|
"review_policy": "Retain first-run model verdicts. Supplemental AI source review may identify evaluator mistakes but must not overwrite the strict result or claim human validation."
|
|
}
|