Measure frozen delivery on 80 source-bound questions per retrieval route
This commit is contained in:
parent
df00d12469
commit
2da46418c7
121
experiments/delivery_contract_v1/build_test.py
Normal file
121
experiments/delivery_contract_v1/build_test.py
Normal file
@ -0,0 +1,121 @@
|
||||
"""Author a source-bound, question-held-out TEST before any backend execution.
|
||||
|
||||
Evaluation-only templates, never imported by production. Gold comes from raw
|
||||
Markdown, not a runtime extractor or its knowledge graph. Repeated profiles and
|
||||
AI-authored gold do not constitute unseen-document or human-reviewed accuracy.
|
||||
"""
|
||||
from datetime import date
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from pathlib import Path
|
||||
import frontmatter
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[2]
|
||||
WIKI=ROOT.parent/'zpwiki'
|
||||
OUT=Path(__file__).parent
|
||||
|
||||
|
||||
def profile_options():
|
||||
used=set()
|
||||
for p in (ROOT/'experiments/user_questions_v1').glob('*.json'):
|
||||
cases=json.loads(p.read_text()).get('cases',[])
|
||||
if isinstance(cases,list):used.update(e['path'] for c in cases for e in c.get('evidence',[]) if 'path' in e)
|
||||
# Current conversational DEV subjects remain outside this new sample.
|
||||
dev={'jan_holp','ondrej_megela','jakub_schwarc','jan_ptak','jan_malinovsky','maksym_zatirka','martin_sarissky','samuel_vasil','samuel_horani'}
|
||||
result=[]
|
||||
for p in sorted((WIKI/'pages/students').glob('*/*/README.md')):
|
||||
relative=str(p.relative_to(WIKI))
|
||||
if relative in used or p.parent.name in dev:continue
|
||||
raw=p.read_text();page=frontmatter.loads(raw)
|
||||
if page.get('published') is False:continue
|
||||
lines=raw.splitlines()
|
||||
headings=[(i,l.strip('# *')) for i,l in enumerate(lines) if re.match(r'^#+\s+(?:Bakalársk|Diplomov|Tímov|Výskumn|Vedeck)',l)]
|
||||
for h,(start,label) in enumerate(headings):
|
||||
if len(re.findall(r'\b20\d{2}\b',label))!=1 or re.search(r'\d/\d',label):continue
|
||||
end=headings[h+1][0] if h+1<len(headings) else len(lines)
|
||||
meetings=[]
|
||||
for i in range(start+1,end):
|
||||
if re.search(r'(?i)stretnut|konzult',lines[i]):
|
||||
m=re.search(r'(\d{1,2})\.\s*(\d{1,2})\.\s*(20\d{2})',lines[i])
|
||||
try:stamp=date(int(m[3]),int(m[2]),int(m[1])).isoformat() if m else None
|
||||
except ValueError:stamp=None
|
||||
meetings.append((i,stamp))
|
||||
first=meetings[0][0] if meetings else end
|
||||
goal_header=next((i for i in range(start+1,first) if re.search(r'(?i)zadanie|ciele',lines[i])),None)
|
||||
if goal_header is None:continue
|
||||
goals=[];in_goals=True
|
||||
for i in range(goal_header+1,first):
|
||||
text=lines[i].strip('# *: ')
|
||||
if re.search(r'(?i)^(?:možné |predbežné )?(?:zadanie|ciele)\b',text):in_goals=True
|
||||
elif lines[i].strip().endswith(':') or lines[i].startswith('#'):in_goals=False
|
||||
if in_goals and re.match(r'^\s*(?:[-*]|\d+[.)])\s+',lines[i]):goals.append(i)
|
||||
if not 1<=len(goals)<=8:continue
|
||||
complete=[m for m in meetings if m[1]]
|
||||
if not complete:continue
|
||||
chosen=max(complete,key=lambda m:m[1]);mi,stamp=chosen
|
||||
mend=next((i for i,_ in meetings if i>mi),end)
|
||||
mode=None;state=[];tasks=[]
|
||||
for i in range(mi+1,mend):
|
||||
text=lines[i].strip('# *: ').casefold()
|
||||
if text.startswith('stav'):mode='state';continue
|
||||
if text.startswith(('úlohy','ulohy')):mode='tasks';continue
|
||||
if text.startswith(('zásobník','zasobnik')):mode=None
|
||||
if re.match(r'^\s*[-*]\s+',lines[i]):
|
||||
if mode=='state':state.append(i)
|
||||
if mode=='tasks':tasks.append(i)
|
||||
if not 1<=len(state)<=4 or not 1<=len(tasks)<=4:continue
|
||||
result.append({'name':str(page['title']).strip(),'path':relative,'label':label,'lines':lines,
|
||||
'sha256':hashlib.sha256(raw.encode()).hexdigest(),'headings':headings,'goals':goals,
|
||||
'state':state,'tasks':tasks,'meeting':mi,'date':stamp})
|
||||
random.Random(20261110).shuffle(result)
|
||||
selected=[];seen=set()
|
||||
for p in result:
|
||||
if p['path'] not in seen:selected.append(p);seen.add(p['path'])
|
||||
if len(selected)==10:return selected
|
||||
raise ValueError('Not enough independently source-bindable profiles; do not manufacture gold')
|
||||
|
||||
|
||||
def build():
|
||||
target=OUT/'test.json'
|
||||
if target.exists():raise ValueError('Never overwrite a prepared TEST')
|
||||
cases=[]
|
||||
def ref(p,n,requirement,end=None):
|
||||
end=n if end is None else end
|
||||
return {'path':p['path'],'sha256':p['sha256'],'url':'https://zp.kemt.fei.tuke.sk/'+p['path'].removeprefix('pages/').removesuffix('/README.md'),
|
||||
'start':n+1,'end':end+1,'quote':'\n'.join(p['lines'][n:end+1]),'requirements':[requirement],
|
||||
'retrieval_terms':[[re.sub(r'^\s*(?:[-*]|\d+[.)])\s+','',p['lines'][n])]]}
|
||||
def add(category,q,evidence,status='ok',requirements=()):
|
||||
assert len(q)<=500
|
||||
cases.append({'id':f'C{len(cases)+1:03d}','category':category,'question':q,
|
||||
'expected_status':status,'evidence':evidence,'response_requirements':list(requirements)})
|
||||
for p in profile_options():
|
||||
name,label=p['name'],p['label'];year=re.search(r'\b20\d{2}\b',label)[0]
|
||||
y,m,d=p['date'].split('-');day=f'{int(d)}.{int(m)}.{y}'
|
||||
heading=next(i for i,l in p['headings'] if l==label)
|
||||
work=[ref(p,heading,f'{name}: explicitly identify record {label}. Work year {year} is not study start or completion.')]
|
||||
goals=[ref(p,n,f'{name}, {label}: include this goal as assigned or proposed, never already achieved: {p["lines"][n]}') for n in p['goals']]
|
||||
state=[ref(p,n,f'{name}, {label}, record {day}: explain this recorded state, preserving uncertainty/negation and without claiming graduation: {p["lines"][n]}') for n in p['state']]
|
||||
tasks=[ref(p,n,f'{name}, {label}, record {day}: planned follow-up, not an achieved result: {p["lines"][n]}') for n in p['tasks']]
|
||||
marker=ref(p,p['meeting'],f'Explicit record date is {day}, unrelated to work year {year}.')
|
||||
add('record_identity',f'U osoby {name} uveď typ a evidovaný rok záznamu „{label}“.',[*work])
|
||||
inventory=[ref(p,n,f'Include this distinct record of {name}: {l}. A heading is not evidence of completed study.') for n,l in p['headings']]
|
||||
add('inventory',f'Vypíš všetky evidované práce a projekty osoby {name}. Pri každom rozlíš typ a rok podľa nadpisu.',inventory)
|
||||
add('goals',f'Čo bolo zadaním práce „{label}“ osoby {name}? Vysvetli všetky doložené ciele, nie domnelé výsledky.',goals)
|
||||
add('dated_state',f'Ako vyzeral zaznamenaný stav osoby {name} pri práci „{label}“ na stretnutí {day}?',[marker,*state])
|
||||
add('dated_plans',f'Aké ďalšie úlohy mala osoba {name} pri práci „{label}“ podľa zápisu {day}? Rozlišuj ich od hotovej práce.',[marker,*tasks])
|
||||
add('latest_complete_record',f'Kedy je posledný záznam s úplným dátumom pri práci „{label}“ osoby {name}?',[marker],requirements=[
|
||||
'Do not complete a partial date with a guessed year. Qualify latest as latest fully dated record; undated entries may not be ordered.'])
|
||||
add('assignment_vs_record',f'Porovnaj zadanie práce „{label}“ osoby {name} s tým, čo je pri nej zaznamenané ako stav na stretnutí {day}. Nezamieňaj plán za výsledok.',[*goals,marker,*state])
|
||||
assert not any('1997' in l for _,l in p['headings'])
|
||||
add('absent_work_year',f'Aké práce má osoba {name} evidované v roku 1997?',[*inventory],status='no_answer',requirements=[
|
||||
'No record from other work years may substitute the explicit 1997 filter. Absence in indexed records does not prove no such work exists anywhere.'])
|
||||
value={'schema':1,'split':'TEST','authorship':'AI-authored against raw Markdown; no human gold review claimed',
|
||||
'sampling':{'seed':20261110,'profiles':10,'questions_per_profile':8,
|
||||
'exclusions':'Previously used user_questions_v1 gold profiles and current conversational DEV profiles.',
|
||||
'limits':'Question-held-out public corpus; repeated profiles, known task families, selected structured profiles. Not an unseen-document population estimate.'},'cases':cases}
|
||||
OUT.mkdir(parents=True,exist_ok=True);target.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print(json.dumps({'cases':len(cases),'profiles':[p['name'] for p in profile_options()]},ensure_ascii=False))
|
||||
|
||||
if __name__=='__main__':build()
|
||||
197
experiments/delivery_contract_v1/judge_calibration.json
Normal file
197
experiments/delivery_contract_v1/judge_calibration.json
Normal file
@ -0,0 +1,197 @@
|
||||
{
|
||||
"scope": "Synthetic DEV judge calibration, not TEST results",
|
||||
"evaluator_sha256": "e859e1eec8fc2a2a6993a9fce57e875e3fb11d3a93125e801cb0ca178b572461",
|
||||
"script_sha256": "fd1e1cf5d8a54ab65398f0e345bbbbd8e33f4ad9fcd2e268a1e7ced3fe9e2507",
|
||||
"passed": 8,
|
||||
"total": 8,
|
||||
"results": [
|
||||
{
|
||||
"id": "planned_supported",
|
||||
"expected_accepted": true,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": true,
|
||||
"coverage": true,
|
||||
"citations": true,
|
||||
"appropriate_response": true,
|
||||
"clarity": 3,
|
||||
"critical_errors": [],
|
||||
"reasons": [],
|
||||
"proofs": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "phase_mutation",
|
||||
"expected_accepted": false,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": false,
|
||||
"coverage": false,
|
||||
"citations": false,
|
||||
"appropriate_response": false,
|
||||
"clarity": 3,
|
||||
"critical_errors": [
|
||||
"wrong_work",
|
||||
"invented_fact",
|
||||
"unsupported_time"
|
||||
],
|
||||
"reasons": [
|
||||
"wrong_work",
|
||||
"invented_fact",
|
||||
"unsupported_time"
|
||||
],
|
||||
"proofs": [
|
||||
{
|
||||
"code": "wrong_work",
|
||||
"answer_quote": "Eva už úspešne vyhodnotila model BERT.",
|
||||
"source_path": "pages/students/2020/eva/README.md",
|
||||
"source_quote": "## Bakalárska práca 2024\nStretnutie 5.5.2026\nStav:\n- Implementovaný model BERT."
|
||||
},
|
||||
{
|
||||
"code": "invented_fact",
|
||||
"answer_quote": "Eva už úspešne vyhodnotila model BERT.",
|
||||
"source_path": "pages/students/2020/eva/README.md",
|
||||
"source_quote": "# Diplomová práca 2027\nCiele:\n- Vyhodnoťte model BERT."
|
||||
},
|
||||
{
|
||||
"code": "unsupported_time",
|
||||
"answer_quote": "Eva už úspešne vyhodnotila model BERT.",
|
||||
"source_path": "pages/students/2020/eva/README.md",
|
||||
"source_quote": "# Diplomová práca 2027\nCiele:\n- Vyhodnoťte model BERT."
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "nested_work_supported",
|
||||
"expected_accepted": true,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": true,
|
||||
"coverage": true,
|
||||
"citations": true,
|
||||
"appropriate_response": true,
|
||||
"clarity": 3,
|
||||
"critical_errors": [],
|
||||
"reasons": [],
|
||||
"proofs": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "work_mutation",
|
||||
"expected_accepted": false,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": false,
|
||||
"coverage": false,
|
||||
"citations": false,
|
||||
"appropriate_response": false,
|
||||
"clarity": 3,
|
||||
"critical_errors": [
|
||||
"wrong_work",
|
||||
"unsupported_time"
|
||||
],
|
||||
"reasons": [
|
||||
"wrong_work",
|
||||
"unsupported_time"
|
||||
],
|
||||
"proofs": [
|
||||
{
|
||||
"code": "wrong_work",
|
||||
"answer_quote": "Pri diplomovej práci 2027 je zaznamenaná implementácia BERT.",
|
||||
"source_path": "pages/students/2020/eva/README.md",
|
||||
"source_quote": "## Bakalárska práca 2024\nStretnutie 5.5.2026\nStav:\n- Implementovaný model BERT."
|
||||
},
|
||||
{
|
||||
"code": "unsupported_time",
|
||||
"answer_quote": "Pri diplomovej práci 2027 je zaznamenaná implementácia BERT.",
|
||||
"source_path": "pages/students/2020/eva/README.md",
|
||||
"source_quote": "Stretnutie 15.5.\nStav:"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "actor_mutation",
|
||||
"expected_accepted": false,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": false,
|
||||
"coverage": true,
|
||||
"citations": true,
|
||||
"appropriate_response": true,
|
||||
"clarity": 3,
|
||||
"critical_errors": [
|
||||
"wrong_person"
|
||||
],
|
||||
"reasons": [
|
||||
"wrong_person"
|
||||
],
|
||||
"proofs": [
|
||||
{
|
||||
"code": "wrong_person",
|
||||
"answer_quote": "Alena Nová úspešne obhájila diplomovú prácu v roku 2024.",
|
||||
"source_path": "pages/students/2020/alena/README.md",
|
||||
"source_quote": "Peter Nový úspešne obhájil diplomovú prácu v roku 2024."
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "actor_abstention_supported",
|
||||
"expected_accepted": true,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": true,
|
||||
"coverage": true,
|
||||
"citations": true,
|
||||
"appropriate_response": true,
|
||||
"clarity": 4,
|
||||
"critical_errors": [],
|
||||
"reasons": [],
|
||||
"proofs": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "partial_year_supported",
|
||||
"expected_accepted": true,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": true,
|
||||
"coverage": true,
|
||||
"citations": true,
|
||||
"appropriate_response": true,
|
||||
"clarity": 4,
|
||||
"critical_errors": [],
|
||||
"reasons": [],
|
||||
"proofs": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "partial_year_inference",
|
||||
"expected_accepted": false,
|
||||
"passed": true,
|
||||
"grade": {
|
||||
"correct": false,
|
||||
"coverage": true,
|
||||
"citations": false,
|
||||
"appropriate_response": true,
|
||||
"clarity": 3,
|
||||
"critical_errors": [
|
||||
"unsupported_time"
|
||||
],
|
||||
"reasons": [
|
||||
"unsupported_time"
|
||||
],
|
||||
"proofs": [
|
||||
{
|
||||
"code": "unsupported_time",
|
||||
"answer_quote": "Rozhranie funguje podľa záznamu z 15.5.2024.",
|
||||
"source_path": "pages/students/2020/eva/README.md",
|
||||
"source_quote": "Stretnutie 15.5."
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
21
experiments/delivery_contract_v1/protocol.json
Normal file
21
experiments/delivery_contract_v1/protocol.json
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"purpose": "Source-bound paired request fulfilment TEST; not global or human-verified accuracy.",
|
||||
"method_source": "https://arxiv.org/abs/2309.15217",
|
||||
"test_cases": 80,
|
||||
"paired_trials": 160,
|
||||
"engines": [
|
||||
"rag",
|
||||
"graphrag"
|
||||
],
|
||||
"workers": 2,
|
||||
"question_held_out": true,
|
||||
"unseen_documents": false,
|
||||
"threshold_per_engine": 0.94,
|
||||
"calibration_cases": 8,
|
||||
"no_test_directed_tuning_after_freeze": true,
|
||||
"sections": {
|
||||
"Fixed design before execution": "- 80 new exact questions, 10 profiles, eight categories per profile. Sampling\n seed: 20261110. `build_test.py` reads raw wiki Markdown, not production answers.\n- Categories: record identity, work inventory, assignment, dated state, dated\n plans, latest fully dated record, assignment versus dated state, absent year.\n- Exclude previously used `user_questions_v1` gold profiles and conversational\n DEV subjects. Check exact-question overlap against earlier experiment JSONs.\n- Source paths, SHA256, literal lines and expected requirements are in\n `test.json`. The complete gold is never sent to the answering endpoint.\n- Run each question once through RAG and GraphRAG, with the same configured\n model, temperature, snapshot, limit and grading rules: 160 paired trials.\n- Freeze dataset, evaluator, runtime file hashes, index, knowledge snapshot and\n model configuration before the first TEST request. No TEST-directed tuning,\n answer regeneration or replacement of failed trials. Two concurrent workers.\n- Calibrate the separate judge on eight synthetic DEV examples: supported facts\n and mutations of actor, work, planned/result phase and incomplete dates.\n- Success requires supported facts, all requested requirements, supporting\n citations, appropriate response behaviour and clarity of at least 3/4.\n Rejection of an answerable question is a failure, not successful caution.\n- Candidate and selected evidence coverage are reported separately from final\n answer quality. The evaluator reads original source windows independently.\n- Target: at least 94% per retrieval route on this TEST. Keep all failures in\n the denominator, including execution and judge failures. Report critical\n substitutions and false refusals separately. Do not pool the paired routes as\n 160 independent questions.",
|
||||
"Known limitations": "Gold is AI-authored and checked against raw source bytes; it has no human review.\nThe judge uses the same configured model in a separate call, so correlated errors\nremain possible. Profiles repeat across questions, categories are structured and\nknown, and the public corpus may have been seen before. This is a question-held-out\ntest, not an unseen-document or broad population estimate. Wilson intervals are\ndescriptive IID intervals, not cluster-adjusted guarantees.\n\nConversation, transport and adversarial scope tests belong to known regressions;\ntheir success must not be added to this TEST accuracy. Real Claude browser\nverification remains a separate manual step. A backend audit cannot enforce a\nlater UI rewrite of its accepted answer.",
|
||||
"Artifacts": "`test_manifest.json` binds the frozen setup. `test_results.jsonl` preserves the\nfirst run, and `test_results.summary.json` reports its aggregate outcome. Failed\ncases can inform a later DEV cycle; a corrected rerun is reproduction, not fresh\nindependent TEST accuracy."
|
||||
}
|
||||
}
|
||||
44
experiments/delivery_contract_v1/review_notes.json
Normal file
44
experiments/delivery_contract_v1/review_notes.json
Normal file
@ -0,0 +1,44 @@
|
||||
{
|
||||
"scope": "Post-run observations on the frozen run, not regrading or replacement scores. Frozen runtime and original verdicts remain unchanged.",
|
||||
"observations": [
|
||||
{
|
||||
"cases": [
|
||||
"C001/rag",
|
||||
"C001/graphrag"
|
||||
],
|
||||
"type": "confirmed_false_refusal",
|
||||
"observation": "The explicit work/type/year question was rejected at scope/wrong_work_or_meeting. Read-only retrieval diagnosis found the correct thesis, year and heading in raw graph facts, while graph_context kept type/year but discarded the heading required by the quoted-label scope check. The binding needs to preserve trusted work identity; do not waive the scope guard."
|
||||
},
|
||||
{
|
||||
"cases": [
|
||||
"C011/rag",
|
||||
"C011/graphrag"
|
||||
],
|
||||
"type": "confirmed_requirement_failure",
|
||||
"observation": "The question requests goals, not supposed results. The delivered answer says goals are missing and supplies results. Raw Markdown explicitly contains Predbežné zadanie - úlohy na semester and three assignment bullets. Required repairs concern source section recognition, exclusion/negation in the plan, and availability checks beyond selected evidence."
|
||||
},
|
||||
{
|
||||
"cases": [
|
||||
"C004/graphrag"
|
||||
],
|
||||
"type": "unsupported_judge_verdict",
|
||||
"observation": "The unsupported_time proof compares answer Podľa zápisu zo stretnutia 2024-12-12 against source Stretnutie 12.12.2024. These are the same explicit record date. This proof does not establish an invented date. Original score is preserved."
|
||||
},
|
||||
{
|
||||
"cases": [
|
||||
"C010/rag",
|
||||
"C010/graphrag"
|
||||
],
|
||||
"type": "unsupported_judge_proof",
|
||||
"observation": "The unsupported_time proof quotes Bakalárska práca 2024 from both answer and source. An identical source work-year label does not itself demonstrate an unsupported time claim. This is not a full manual regrade of the answers."
|
||||
},
|
||||
{
|
||||
"cases": [
|
||||
"C003/rag"
|
||||
],
|
||||
"type": "disputed_judge_proof",
|
||||
"observation": "The wrong_phase proof compares a stated Ciel with the same goal rendered under Zadanie a ciele as zadanie. That quoted pair alone does not show a plan upgraded into an achieved result. The answer also contains unrequested results, which still needs separate fulfilment review."
|
||||
}
|
||||
],
|
||||
"conclusion": "Do not infer true general accuracy, zero critical errors or readiness from this automatic judge. Calibration on eight synthetic scenarios did not establish reliability for this diverse TEST. Report raw conservative pass counts and false refusals, retain uncertain grades, and require further source-based review."
|
||||
}
|
||||
9
experiments/delivery_contract_v1/run_validation.json
Normal file
9
experiments/delivery_contract_v1/run_validation.json
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
"frozen_code_commit": "df00d124697cd4703e79579287c1f50bf5fee255",
|
||||
"runtime_unchanged_through_end": true,
|
||||
"evaluator_unchanged_through_end": true,
|
||||
"complete": true,
|
||||
"trials": 160,
|
||||
"snapshot_valid": true,
|
||||
"raw_scores_are_not_general_accuracy": true
|
||||
}
|
||||
4982
experiments/delivery_contract_v1/test.json
Normal file
4982
experiments/delivery_contract_v1/test.json
Normal file
File diff suppressed because it is too large
Load Diff
152
experiments/delivery_contract_v1/test_manifest.json
Normal file
152
experiments/delivery_contract_v1/test_manifest.json
Normal file
@ -0,0 +1,152 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"created_utc": "2026-11-10T07:06:29.778840+00:00",
|
||||
"split": "TEST",
|
||||
"dataset_sha256": "bdb40c23d7df909b4a3ff6cf331df3566cd129b345e1768bc67fb37ef2647af2",
|
||||
"evaluator_sha256": "e859e1eec8fc2a2a6993a9fce57e875e3fb11d3a93125e801cb0ca178b572461",
|
||||
"runtime_files": {
|
||||
"app/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"app/main.py": "38ae3f77e8a0ddec414459da1ec17d5247d2f9457bc7ceb2af17d442658a1d28",
|
||||
"app/routes.py": "55b2178834dc0e98cf832c4c87ef07304a99b0ebffc9d4b41093fbfaaed10235",
|
||||
"app/security.py": "ac1dc6c637e007cffb1997007e9b5ec2a398d4719349a9177fe551ca6422e6e0",
|
||||
"packages/claude/client.py": "678607f17b6ab9c080ca97552b6f698c4f2a34e2a9fffec945b995e1f282cc87",
|
||||
"packages/claude/graphrag/SKILL.md": "7d113f5b24fc44f94257e17e111447fb9dd6bd57a62cf12b89a6101166c0f8c0",
|
||||
"packages/claude/graphrag/scripts/query.py": "463460cd93cf2a584156df6aa571bcf98297e5a1a7258581d02b084b1e1a6180",
|
||||
"packages/claude/mcp_server.py": "a271586d892a922ce84070e8b782f301d8ea9e2b0204da54205427c774956ef1",
|
||||
"packages/claude/oauth_provider.py": "85e00facfe9c772b174b57f125986a329e3b1bc99f72f4a7f553b763fbea166b",
|
||||
"packages/claude/rag/SKILL.md": "2261d54ef32824c3225c25f58a2dd9f60377df8c19600b857966f5ae33452bcf",
|
||||
"packages/claude/rag/scripts/query.py": "f482f2dc53430cbb383bce4fcc1cccaf580fc7b5a0c885a217fc40cf53713277",
|
||||
"tools/answers/__init__.py": "8b8ab7f996121e5e5b1b594cf08f4c3b4f16f1b434b74276c803a3a83681d2cb",
|
||||
"tools/answers/answer_coverage.py": "746ab3b89ba65f263b63037be55ee02ee25aaaf685dae6e95f7bd6a5536fc6fa",
|
||||
"tools/answers/answer_delivery.py": "1a5adbd545d03457b3ef3c5e5d9c8552a165658faf025e49ba1a2b9ed8dc6b91",
|
||||
"tools/answers/answer_errors.py": "9fa03bfeb1548596341cb2cb619af623811a98af807f0489b4f0188af91bee06",
|
||||
"tools/answers/answer_format.py": "533cfe3a30c835d22441d877df63fe08e366d01ed6d15474fe6fcff26bde259b",
|
||||
"tools/answers/checked_generation.py": "e36def33fb83e5c973fa632685b4782fa411a7eeb7151ef8c14007f58fa1f375",
|
||||
"tools/answers/content_selection.py": "8e09f4423f2fd3b253f5779f62849b11b9570a112a472ea07392468901e3d80e",
|
||||
"tools/answers/context_policy.py": "526f5de0b9112e23abd76e403799f156d0150c83f03577251af23d1685991327",
|
||||
"tools/answers/conversation.py": "a32243add4a59248d8eef92da79de6aa1e200b04e614a27aeb0e2fdb8ae43ce9",
|
||||
"tools/answers/delivery_contract.py": "c69ffe0d10774dfb8c7e39e8dfa2885255239cbab151ca5012e7d5168fe329b7",
|
||||
"tools/answers/evidence_selection.py": "1007a85b8838e1eb3c34ee610a4f81dfff57410a90213e02b8ccde90a0fedbd2",
|
||||
"tools/answers/graph_answer_prompt.txt": "9f61bf69c8787f9fcf650c0228a30dfed5f18a870c9980c6ef892297ebe6013a",
|
||||
"tools/answers/graph_context.py": "5b14150b7f6903cd92c004d56943d949167be8859acf4f97aed14a713e59faff",
|
||||
"tools/answers/model_client.py": "f595eb1b594364faff21ce2a6842c8c7e40a5ca4e6a2c3085cdc4a4bae28ae04",
|
||||
"tools/answers/notices.py": "d226b0476c9fe27c6ae86f0500ca63ea376f3496da3019b48f00736d2259e4ab",
|
||||
"tools/answers/presentation.py": "954c65288ddc3c8f873fec3ef99176adb70785806043ac1814c7a01a267b4aad",
|
||||
"tools/answers/question_intent.py": "c8d2445f93d430eb821bd73fa566ce73ba698e7e426387002d132e73cb524acc",
|
||||
"tools/answers/question_plan.py": "f33d0c749f5f71539a5e3d35904764cf3e432fc19e28abf5d05ff1ec24abd547",
|
||||
"tools/answers/rag_evidence.py": "351b99eb5757991696dbe10dcb794cf02aea7c2232c2b7eabab4e3bf1a7ad687",
|
||||
"tools/answers/rag_metadata.py": "a16486932e66aed571fd25cb59c5656892613488ea90afe80c41b0c1d15afcf1",
|
||||
"tools/answers/rag_preparation.py": "98468b111c9f8f6d423f3c6a07300c3ad7a7e9fef5bae1d1385b91a04771e069",
|
||||
"tools/answers/recommendations.py": "7a27c01ad24d9b8dc78d69c2b95f886deed544ee24d59055a059b52d544f6574",
|
||||
"tools/answers/reference_links.py": "eb59206bcb3b79912daafcd591e7adb504ec592e61ea32e4e133cb6a65fe0b57",
|
||||
"tools/answers/render_answer.py": "953cae2cdbdbf05a1b0c1f2746c4dc5f0d85c9bcfd02c105df38a3ff1d2451af",
|
||||
"tools/answers/synthesis.py": "0188b6354279cb76819c95a0527cff38bc6b616c4ea465a4bfc935edf66b536b",
|
||||
"tools/answers/verified_rag.py": "b2e9777b269f2b19c16ad9e1aa86644f09a9ad96489358a1294e51d21025611e",
|
||||
"tools/graphrag/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"tools/graphrag/build_graphrag.py": "67c5b994ec1be07eadbeaebfad73aa21fbb88e2a28cee805af0034d7cd160b8b",
|
||||
"tools/graphrag/graph_db.py": "5e6364d72cd645087c183633219b4771210c6c3f222946ff4c8e6c512b9b62e5",
|
||||
"tools/graphrag/graph_search.py": "eff65f570d3a52a9cb4132db8f3c4b59dc08840001ae1bfbab6c7ab5eda09bb0",
|
||||
"tools/graphrag/graphrag.py": "2ad7fc728761558d294399d52d5e2e66f6c57e1a88dc1aedf4e2e7ff6375fdcc",
|
||||
"tools/graphrag/neo4j_smoke.py": "fb5244c72c29ef2acf96d1e7f54335ff168eed4119db1692025eb1b99194b7bb",
|
||||
"tools/graphrag/publication.py": "c804b447d02796fb4d53d4b37b843f1ad0d7c259d286b2eb14989ca659184682",
|
||||
"tools/knowledge/__init__.py": "905a19d7cf5d95379800b2ad917ad759fa376f23d2f8a244527b9dc187555019",
|
||||
"tools/knowledge/academic.py": "fb91eb0923cf752c91d1247899df4c0aa0a5baadb913c5d48384f09eb89a3ab7",
|
||||
"tools/knowledge/build.py": "4666b1cd2d42533b7289f5e2ecaac11c5cc16e09f3ea4b9a2714d17cb8c2f9f6",
|
||||
"tools/knowledge/concepts.py": "de0d005ab298c3d86020b0cb09fc695852303d8a5f411d0a3828b58131713b8d",
|
||||
"tools/knowledge/evidence.py": "b0eb7d3e0f6a97083829a9940b3a64dade96d4f98435eb5ebfb43adaf10fac18",
|
||||
"tools/knowledge/extraction.py": "c1819afc5072a5999c985b4547809db997ecf3e1112b031534ae34ed295bd106",
|
||||
"tools/knowledge/graph_answers.py": "fd599b7ff65485ae11e77022cc007b99e5bd3b533383cc64dc5b7aa3dabf81fa",
|
||||
"tools/knowledge/graph_projection.py": "dac3058008fb43c833d239b6a7ced27ef5419004cc768b246741eb2e1a69b12a",
|
||||
"tools/knowledge/local_graph.py": "58af4bd5b888756728739e26d8bc9f4cba1aaf75853c17e11453d4f38fa29f1e",
|
||||
"tools/knowledge/planned_answers.py": "947bc044f8a4942023d7b6d9a7e3a52464bdbca386908e2982daf1c84f789240",
|
||||
"tools/knowledge/publication.py": "a5a29af22fd73a57f53da47c687200f0ad392d48ce94a23250f00c7a53731148",
|
||||
"tools/knowledge/query.py": "e3952189c2ad098ace9be40dd360d52fefaad9d93f09520496dda7336e594a27",
|
||||
"tools/knowledge/sources.py": "709c2201288a920e99578fe4e58e191a742deb5f15170104820680d156280c69",
|
||||
"tools/knowledge/store.py": "fa22cd357df8c3cde4cd1b6a4f0a9b78d12721f3970ec896584205f8a5d5d7c1",
|
||||
"tools/knowledge/text_retrieval.py": "946911f6b1aac8ccf1861c33b75b574df1deca810787b3bd0b2c708a9dd20f71",
|
||||
"tools/rag/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"tools/rag/build_chunks.py": "037f80a423eba26258a8b1f24e8d9129ce4f043568b7db2574a80d01c5ca32ee",
|
||||
"tools/rag/build_sqlite_index.py": "2cb5944543c77d6943fba9f1f7ce8d3c50340bf8e6a4063105e11dbdb608ace0",
|
||||
"tools/rag/common.py": "c325b75d50b0cac6f4b780ad0b2a641b8478b9ba6a71680837fabedb947530fe",
|
||||
"tools/rag/embedding_cache.py": "4c8faa262c59f85842747ba189fa35c1e4d0a4ce2fa5337bd42f498130022387",
|
||||
"tools/rag/embedding_utils.py": "768798b43ca7c99f672ec9e957fb5c3d1c9208409c728cb9af6ae74d31d7d48c",
|
||||
"tools/rag/llm_context.py": "8f195ff16639de4379c41108f7a3a456d90d985307ebde3691c1406b3f76dfa5",
|
||||
"tools/rag/rag_document_expansion.py": "13ef2db80b6bae517cd46feba6a8096a64399019c5b2dd2a2d5ca1ebcdfbf638",
|
||||
"tools/rag/rag_query_evidence.py": "af2dbb7ce94ae625ffc8e01040541b51449b8bd923a4da72027445b779647571",
|
||||
"tools/rag/rag_supplemental_expansion.py": "7c26faf29fcc08ba480d7f374d17aa64b355ae87d9cc8bbaee02afb45c54320e",
|
||||
"tools/rag/rag_utils.py": "fe42c7f7dc2b1679e0d619b29bd3f087c8e1349da1f45b39b2881eb7ea7f1b4a",
|
||||
"tools/rag/rebuild_index.py": "5ee0159f7242d513377824cba6bff8c7d543d4a9c0001dc51499386412f4a3bf",
|
||||
"tools/rag/scan_zpwiki.py": "1a24dcd7cc4063b9c398c81619e87775af07a2e0fa5ad824110c528247123e86",
|
||||
"tools/rag/search_core.py": "f4757c197c1a1def3f1ab351cef7022968331c8c2ac57345015bcc30c6bfece1",
|
||||
"tools/rag/search_db.py": "0356e00c6122b746edee86f28ce86543cfd26e40f55cae31f73feded9be14544",
|
||||
"tools/rag/search_scope.py": "360a1f17a3fe341f829dd7685b253dc1ce2e286362431b638d39b1fe468d3f79",
|
||||
"tools/rag/search_utils.py": "82ce2567d5837e4d1fa7b40ab91a2f5f7f3c992e33282ef0cf14bc3259110644",
|
||||
"tools/rag/sqlite_schema.py": "e2281d500d05891c5c2f6651ca6c116530c8d0bd063637850243f52fc14e2745",
|
||||
"tools/retrieval/__init__.py": "32038cbcb49f11f78f5fd439df555322a03996a5d9935c4f2f5a959528161771",
|
||||
"tools/retrieval/person_names.py": "b617d556cc523a022c13387334daacc5c165816e87116c531dc13a4c61c00cac",
|
||||
"tools/retrieval/publication.py": "bc5924dcba0dccf7434b277af773c60b212cb521b5a617055d00a456091e4e75",
|
||||
"tools/retrieval/question_constraints.py": "d4847b75ab133a8303c5b1b454d9b70bd358dd58c08808c7943d1f31bed44d69",
|
||||
"tools/retrieval/retrieval_contract.py": "3478e02c827d169fbecedd6f3cc11218e2636abed5a55bde416e593fc47d5368",
|
||||
"tools/retrieval/text_budget.py": "cb676f60e827234ad168df86135871f84a4420c77bc1ccfb8d147879c4b84dcd",
|
||||
"tools/retrieval/work_selector.py": "bd89164cbe3a5f9cb04a4c683d1781cfd1ee319c62bf0936421a96748bdaf9c1",
|
||||
"tools/runtime/__init__.py": "684ad0cd460df5746f5785326ba7fcdbb017305b2317a96fbd1272d95f6dc2b2",
|
||||
"tools/runtime/answer_health.py": "a02f5070d01c691db19337e5278e600fa686ea8bd83fa607620a35b81ece3d1c",
|
||||
"tools/runtime/bootstrap_indexes.py": "f032e47390b14138d2f34ae65af68096364fe4eccd1dd3ddc0568806dea3af36",
|
||||
"tools/runtime/report_files.py": "f54913b5b2e4465c11c40212b3fe0203cbad84007ffe46281753f485bc9d686d",
|
||||
"tools/runtime/runtime_assets.py": "93d9ba92200e96d17f90e7171fff0b43ffe632c05dcfe446838fa87ad84db0bf",
|
||||
"tools/runtime/runtime_health.py": "4685a46a0d8468db60fdb7bb3df3dbc1700c724f3dc7226ae431b2bd4b5272e4",
|
||||
"tools/runtime/tool_health.py": "bc661bd32a6e6e6694612aa06879d5582a891053175ca6ac7ce2878bcb74701c",
|
||||
"tools/sync/__init__.py": "2e0323ee92a26a8487ed7b60abd6c47f8c66755bc695c5e76b845503f34fb1f1",
|
||||
"tools/sync/index_state.py": "a29a1daef5e1a0b52a9cfbadd1469d3eb8ab53e25939342b2f2df8941ffaf689",
|
||||
"tools/sync/sync_indexes.py": "168d973d514a1220347f28d3fce079ee5a188ac283319cd8cda146a409cae9b8",
|
||||
"tools/sync/sync_jobs.py": "d1ec1f05a9ffffa650b953c3217b39b84aef72add04a5d490eee9e7f194a290a",
|
||||
"tools/sync/wiki_git.py": "adaea932acfea3829bbc1f12ea6fbf59948d8ebee530f6351b013ad68d78d1ce"
|
||||
},
|
||||
"index_version": {
|
||||
"version": "815b67d4b9c54cbcb5d6bd87251fd2ef",
|
||||
"git_revision": "c3ec9f611e645a2fe14fe66d518eb0924bf9bb40",
|
||||
"created_at_utc": "2026-11-09T20:49:43.773353+00:00",
|
||||
"source_sha256": "201798bf276106986e7ef01b4d6357960cb495d09c9855b7df922c1df4fb35a7",
|
||||
"documents_sha256": "8b7eea54f6233cdc9d16610296d32b9336ab0cbaa4b881872a088c05464960be",
|
||||
"chunks_sha256": "8357d1e0641206990d26827b94f37916661595261f368e30f828bccd03171c95",
|
||||
"embedding_model": "intfloat/multilingual-e5-small",
|
||||
"publication_policy": "explicit-false-hidden-v1",
|
||||
"embedding_fingerprint": "d066fb9a266a1a0c77c65da75dcf538cfab5c87d5c605ff26b1fd2da8a12aa92"
|
||||
},
|
||||
"knowledge_version": "9440b8501af042b871e40de3fdce8899",
|
||||
"model": {
|
||||
"provider": "openai_compatible",
|
||||
"model": "model120-fast",
|
||||
"endpoint": "https://ui.tukekemt.xyz/api/chat/completions"
|
||||
},
|
||||
"temperature": 0,
|
||||
"structured_output": "prompt",
|
||||
"limit": 10,
|
||||
"cases": 80,
|
||||
"categories": {
|
||||
"record_identity": 10,
|
||||
"inventory": 10,
|
||||
"goals": 10,
|
||||
"dated_state": 10,
|
||||
"dated_plans": 10,
|
||||
"latest_complete_record": 10,
|
||||
"assignment_vs_record": 10,
|
||||
"absent_work_year": 10
|
||||
},
|
||||
"threshold": 0.94,
|
||||
"answer_retries": 0,
|
||||
"judge_validation_attempts": 2,
|
||||
"previous_exact_question_overlap": [],
|
||||
"judge_calibration": {
|
||||
"sha256": "6bced0cfa548a8547c36aa918c541654c4075609d7eb876c146e0d1462552028",
|
||||
"passed": 8,
|
||||
"total": 8
|
||||
},
|
||||
"limitations": [
|
||||
"Question-held-out on an existing public corpus, not unseen documents or model training data.",
|
||||
"Gold authored and checked against raw Markdown by AI; no human gold review claimed.",
|
||||
"Same configured model judges answers in a separate call; correlated errors remain possible.",
|
||||
"Several questions share source profiles; Wilson intervals are descriptive IID intervals, not a cluster-corrected population guarantee.",
|
||||
"No runtime changes or TEST-directed tuning are allowed after this freeze. A rerun is reproduction."
|
||||
]
|
||||
}
|
||||
321
experiments/delivery_contract_v1/test_results.jsonl
Normal file
321
experiments/delivery_contract_v1/test_results.jsonl
Normal file
File diff suppressed because one or more lines are too long
251
experiments/delivery_contract_v1/test_results.summary.json
Normal file
251
experiments/delivery_contract_v1/test_results.summary.json
Normal file
@ -0,0 +1,251 @@
|
||||
{
|
||||
"manifest_sha256": "3b2b844562879759d09fec8ad2aeaeafc4bd78201db1200906906a44cea022bf",
|
||||
"split": "TEST",
|
||||
"complete": true,
|
||||
"assessment": "Automated source-backed assessment, not human validation or global accuracy.",
|
||||
"rag": {
|
||||
"total": 80,
|
||||
"passed": 27,
|
||||
"success_percent": 33.75,
|
||||
"target_met": false,
|
||||
"wilson_95_percent": [
|
||||
24.35,
|
||||
44.64
|
||||
],
|
||||
"issues": {
|
||||
"delivery_rejected": 30,
|
||||
"answer_rejected": 30,
|
||||
"false_refusal": 31,
|
||||
"correct": 19,
|
||||
"wrong_phase": 2,
|
||||
"judge_unavailable": 4,
|
||||
"unsupported_time": 8,
|
||||
"appropriate_response": 5,
|
||||
"citations": 4,
|
||||
"coverage": 7,
|
||||
"invented_fact": 2,
|
||||
"missing_citation": 2,
|
||||
"missing_requirement": 6,
|
||||
"readability": 2,
|
||||
"unsupported_citation": 1,
|
||||
"wrong_status": 2,
|
||||
"unsupported_number": 3,
|
||||
"wrong_work": 1
|
||||
},
|
||||
"categories": {
|
||||
"absent_work_year": {
|
||||
"total": 10,
|
||||
"passed": 10
|
||||
},
|
||||
"assignment_vs_record": {
|
||||
"total": 10,
|
||||
"passed": 0
|
||||
},
|
||||
"dated_plans": {
|
||||
"total": 10,
|
||||
"passed": 7
|
||||
},
|
||||
"dated_state": {
|
||||
"total": 10,
|
||||
"passed": 4
|
||||
},
|
||||
"goals": {
|
||||
"total": 10,
|
||||
"passed": 2
|
||||
},
|
||||
"inventory": {
|
||||
"total": 10,
|
||||
"passed": 4
|
||||
},
|
||||
"latest_complete_record": {
|
||||
"total": 10,
|
||||
"passed": 0
|
||||
},
|
||||
"record_identity": {
|
||||
"total": 10,
|
||||
"passed": 0
|
||||
}
|
||||
},
|
||||
"retrieval": {
|
||||
"candidate_coverage": {
|
||||
"all_required": 35,
|
||||
"evaluable": 40
|
||||
},
|
||||
"selected_coverage": {
|
||||
"all_required": 13,
|
||||
"evaluable": 31
|
||||
}
|
||||
},
|
||||
"dimensions": {
|
||||
"correct": {
|
||||
"passed": 27,
|
||||
"total": 80,
|
||||
"judge_evaluable": 46
|
||||
},
|
||||
"coverage": {
|
||||
"passed": 39,
|
||||
"total": 80,
|
||||
"judge_evaluable": 46
|
||||
},
|
||||
"citations": {
|
||||
"passed": 42,
|
||||
"total": 80,
|
||||
"judge_evaluable": 46
|
||||
},
|
||||
"appropriate_response": {
|
||||
"passed": 41,
|
||||
"total": 80,
|
||||
"judge_evaluable": 46
|
||||
},
|
||||
"clarity": {
|
||||
"passed": 44,
|
||||
"total": 80
|
||||
}
|
||||
},
|
||||
"false_refusals": 31,
|
||||
"delivery_failure_layers": {
|
||||
"scope": 10,
|
||||
"delivery": 22,
|
||||
"verification": 6,
|
||||
"fulfilment": 10,
|
||||
"answer": 1,
|
||||
"evidence": 3,
|
||||
"retrieval": 1
|
||||
},
|
||||
"critical_error_questions": 19,
|
||||
"synthesis": {
|
||||
"deterministic": 51,
|
||||
"fallback": 12,
|
||||
"accepted": 17
|
||||
},
|
||||
"answer_latency_seconds": {
|
||||
"median": 22.08,
|
||||
"p95": 77.59
|
||||
}
|
||||
},
|
||||
"graphrag": {
|
||||
"total": 80,
|
||||
"passed": 27,
|
||||
"success_percent": 33.75,
|
||||
"target_met": false,
|
||||
"wilson_95_percent": [
|
||||
24.35,
|
||||
44.64
|
||||
],
|
||||
"issues": {
|
||||
"delivery_rejected": 30,
|
||||
"answer_rejected": 30,
|
||||
"false_refusal": 32,
|
||||
"correct": 17,
|
||||
"unsupported_time": 9,
|
||||
"judge_unavailable": 6,
|
||||
"citations": 5,
|
||||
"wrong_work": 1,
|
||||
"wrong_status": 1,
|
||||
"unsupported_number": 4,
|
||||
"appropriate_response": 3,
|
||||
"coverage": 5,
|
||||
"invented_fact": 1,
|
||||
"missing_citation": 2,
|
||||
"missing_requirement": 4,
|
||||
"readability": 1,
|
||||
"wrong_phase": 1
|
||||
},
|
||||
"categories": {
|
||||
"absent_work_year": {
|
||||
"total": 10,
|
||||
"passed": 10
|
||||
},
|
||||
"assignment_vs_record": {
|
||||
"total": 10,
|
||||
"passed": 0
|
||||
},
|
||||
"dated_plans": {
|
||||
"total": 10,
|
||||
"passed": 7
|
||||
},
|
||||
"dated_state": {
|
||||
"total": 10,
|
||||
"passed": 4
|
||||
},
|
||||
"goals": {
|
||||
"total": 10,
|
||||
"passed": 2
|
||||
},
|
||||
"inventory": {
|
||||
"total": 10,
|
||||
"passed": 4
|
||||
},
|
||||
"latest_complete_record": {
|
||||
"total": 10,
|
||||
"passed": 0
|
||||
},
|
||||
"record_identity": {
|
||||
"total": 10,
|
||||
"passed": 0
|
||||
}
|
||||
},
|
||||
"retrieval": {
|
||||
"candidate_coverage": {
|
||||
"all_required": 35,
|
||||
"evaluable": 40
|
||||
},
|
||||
"selected_coverage": {
|
||||
"all_required": 12,
|
||||
"evaluable": 30
|
||||
}
|
||||
},
|
||||
"dimensions": {
|
||||
"correct": {
|
||||
"passed": 27,
|
||||
"total": 80,
|
||||
"judge_evaluable": 44
|
||||
},
|
||||
"coverage": {
|
||||
"passed": 39,
|
||||
"total": 80,
|
||||
"judge_evaluable": 44
|
||||
},
|
||||
"citations": {
|
||||
"passed": 39,
|
||||
"total": 80,
|
||||
"judge_evaluable": 44
|
||||
},
|
||||
"appropriate_response": {
|
||||
"passed": 41,
|
||||
"total": 80,
|
||||
"judge_evaluable": 44
|
||||
},
|
||||
"clarity": {
|
||||
"passed": 43,
|
||||
"total": 80
|
||||
}
|
||||
},
|
||||
"false_refusals": 32,
|
||||
"delivery_failure_layers": {
|
||||
"scope": 10,
|
||||
"fulfilment": 11,
|
||||
"delivery": 22,
|
||||
"verification": 6,
|
||||
"evidence": 3,
|
||||
"retrieval": 1
|
||||
},
|
||||
"critical_error_questions": 17,
|
||||
"synthesis": {
|
||||
"deterministic": 52,
|
||||
"accepted": 17,
|
||||
"fallback": 11
|
||||
},
|
||||
"answer_latency_seconds": {
|
||||
"median": 20.15,
|
||||
"p95": 68.72
|
||||
}
|
||||
},
|
||||
"paired": {
|
||||
"questions": 80,
|
||||
"both_pass": 27,
|
||||
"rag_only": 0,
|
||||
"graph_only": 0
|
||||
},
|
||||
"valid_snapshot": true
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user