72 lines
4.9 KiB
Python
72 lines
4.9 KiB
Python
"""Live DEV calibration: verify that the judge accepts correct source-bound text
|
|
and detects work/actor/phase substitutions. Not part of measured TEST accuracy.
|
|
"""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
|
|
from tools.evaluation.user_questions import judge_answer,digest
|
|
|
|
|
|
def calibrate(output):
|
|
if output.exists():raise ValueError('Calibration result exists; do not overwrite')
|
|
rows=[]
|
|
with tempfile.TemporaryDirectory(prefix='zp-judge-calibration-') as directory:
|
|
wiki=Path(directory)
|
|
raw='---\ntitle: Eva Nová\n---\n# Diplomová práca 2027\nCiele:\n- Vyhodnoťte model BERT.\n## Bakalárska práca 2024\nStretnutie 5.5.2026\nStav:\n- Implementovaný model BERT.\n'
|
|
raw+='Stretnutie 15.5.\nStav:\n- Funguje rozhranie.\n'
|
|
path=wiki/'pages/students/2020/eva/README.md';path.parent.mkdir(parents=True);path.write_text(raw)
|
|
alena=wiki/'pages/students/2020/alena/README.md'
|
|
alena.parent.mkdir(parents=True)
|
|
alena.write_text('---\ntitle: Alena Nová\n---\n# Diplomová práca 2024\nStav:\n- Peter Nový úspešne obhájil diplomovú prácu v roku 2024.\n')
|
|
scenarios=[
|
|
('planned_supported',path,6,'Aké má Eva zadanie diplomovej práce 2027?',
|
|
'Má vyhodnotiť model BERT.','ok','Assignment, not a performed evaluation.',True),
|
|
('phase_mutation',path,6,'Aké má Eva zadanie diplomovej práce 2027?',
|
|
'Eva už úspešne vyhodnotila model BERT.','ok','The source is only an assignment, not completed evaluation.',False),
|
|
('nested_work_supported',path,10,'Čo je zaznamenané o bakalárskej práci Evy Novej 2024?',
|
|
'Pri bakalárskej práci 2024 je zaznamenaná implementácia BERT. Dátum záznamu je 5.5.2026.','ok',
|
|
'Bachelor thesis 2024 is a separate explicit work. Status is a recorded implementation, date is record date.',True),
|
|
('work_mutation',path,10,'Čo je zaznamenané o bakalárskej práci Evy Novej 2024?',
|
|
'Pri diplomovej práci 2027 je zaznamenaná implementácia BERT.','ok',
|
|
'Implementation belongs to Bachelor thesis 2024, never Master thesis 2027.',False),
|
|
('actor_mutation',alena,6,'Obhájila Alena Nová prácu úspešne?',
|
|
'Alena Nová úspešne obhájila diplomovú prácu v roku 2024.','ok',
|
|
'Source defense actor is Peter Nový, not profile owner Alena Nová.',False),
|
|
('actor_abstention_supported',alena,6,'Obhájila Alena Nová prácu úspešne?',
|
|
'Úspešná obhajoba Aleny Novej sa v týchto podkladoch nepotvrdzuje. Uvedená obhajoba sa týka Petra Nového.','no_answer',
|
|
'No evidence of Alena defense; the quoted defense belongs to Peter.',True),
|
|
('partial_year_supported',path,13,'Čo je zaznamenané pri ďalšom stretnutí Evy?',
|
|
'Rozhranie funguje. Dátum záznamu je 15.5., rok nie je uvedený.','ok',
|
|
'Date is explicitly 15.5. WITHOUT year. Work year 2024 cannot complete that record date.',True),
|
|
('partial_year_inference',path,13,'Čo je zaznamenané pri ďalšom stretnutí Evy?',
|
|
'Rozhranie funguje podľa záznamu z 15.5.2024.','ok',
|
|
'Record year is missing; 2024 is work year, not record date.',False),
|
|
]
|
|
for name,p,n,question,text,status,requirement,expected in scenarios:
|
|
relative=str(p.relative_to(wiki));quote=p.read_text().splitlines()[n-1]
|
|
case={'id':name,'question':question,'expected_status':status,'response_requirements':[],
|
|
'evidence':[{'path':relative,'start':n,'end':n,'quote':quote,'requirements':[requirement]}]}
|
|
answer={'answer':text+'\nZdroj: https://example.test/source','status':status,
|
|
'source_urls':['https://example.test/source'],'evidence':[]}
|
|
try:
|
|
grade=judge_answer(case,answer,wiki)
|
|
accepted=all(grade[k] for k in ['correct','coverage','appropriate_response'])
|
|
passed=accepted==expected
|
|
except Exception:
|
|
grade=None;passed=False
|
|
row={'id':name,'expected_accepted':expected,'passed':passed,'grade':grade}
|
|
rows.append(row);print(json.dumps(row,ensure_ascii=False),flush=True)
|
|
value={'scope':'Synthetic DEV judge calibration, not TEST results','evaluator_sha256':digest(Path(__file__).with_name('user_questions.py')),
|
|
'script_sha256':digest(__file__),'passed':sum(r['passed'] for r in rows),'total':len(rows),'results':rows}
|
|
output.parent.mkdir(parents=True,exist_ok=True)
|
|
output.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
|
return value
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--output',type=Path,required=True)
|
|
args=parser.parse_args();result=calibrate(args.output)
|
|
raise SystemExit(result['passed']!=result['total'])
|