"""An evaluator must detect mistakes, not reward a backend's acceptance flag.""" import copy import json from pathlib import Path import pytest from tools.evaluation import user_questions as e def fixture(tmp_path): path=tmp_path/'pages/students/2020/eva/README.md';path.parent.mkdir(parents=True) path.write_text('---\ntitle: Eva Nová \npublished: true\n---\n# Diplomová práca 2026\nStav:\n- Implementovaný model BERT.\n') gold={'path':str(path.relative_to(tmp_path)),'sha256':e.digest(path),'url':'https://example.test/eva', 'start':7,'end':7,'quote':'- Implementovaný model BERT.', 'requirements':['BERT implementation, not graduation.'],'retrieval_terms':[['BERT']]} case={'id':'T001','category':'result','question':'Čo vytvorila Eva Nová?', 'expected_status':'ok','evidence':[gold],'response_requirements':[]} row={'id':'E1','person':'Eva Nová','source_path':gold['path'],'source_url':gold['url'], 'line':7,'end_line':7,'text':gold['quote'],'work_label':'Diplomová práca 2026'} answer={'status':'ok','accepted_for_delivery':True,'answer':'Implementovala model BERT.\nZdroj: '+gold['url'], 'source_urls':[gold['url']],'evidence':[row], 'retrieval_diagnostics':{'candidate_ids':['E1'],'strategy':'content_checkpoint'}} grade={'correct':True,'coverage':True,'citations':True,'appropriate_response':True,'clarity':4,'critical_errors':[],'reasons':[]} return case,answer,{'E1':row},grade def test_gold_is_verified_against_original_bytes_and_lines(tmp_path): case,_,_,_=fixture(tmp_path) data={'cases':[case]} assert not e.audit_dataset(data,tmp_path) case['evidence'][0]['quote']='Made up text' assert ('T001','wrong_source_lines') in e.audit_dataset(data,tmp_path) def test_gold_duplicate_or_changed_source_is_never_silently_accepted(tmp_path): case,_,_,_=fixture(tmp_path) duplicate=copy.deepcopy(case);duplicate['id']='other' case['evidence'][0]['sha256']='old' issues=e.audit_dataset({'cases':[case,duplicate]},tmp_path) assert ('T001','source_changed') in issues and ('other','duplicate_case') in issues def test_correct_source_owner_whitespace_is_not_a_false_failure(tmp_path,monkeypatch): case,answer,index,grade=fixture(tmp_path) monkeypatch.setattr(e,'judge_answer',lambda *a:grade) score=e.evaluate(case,answer,index,tmp_path) assert score['passed'] and score['retrieval']['candidate_coverage']['all_required'] assert score['retrieval']['selected_coverage']['all_required'] @pytest.mark.parametrize('change,issue',[ ({'answer':'[x](/made-up)'},'unsupported_citation'), ({'source_urls':[]},'missing_citation'), ({'status':'no_answer'},'false_refusal'), ({'accepted_for_delivery':False},'delivery_rejected'), ]) def test_acceptance_or_a_friendly_judge_cannot_waive_mechanical_errors(tmp_path,monkeypatch,change,issue): case,answer,index,grade=fixture(tmp_path);answer.update(change) monkeypatch.setattr(e,'judge_answer',lambda *a:grade) score=e.evaluate(case,answer,index,tmp_path) assert not score['passed'] and issue in score['issues'] @pytest.mark.parametrize('field,value,issue',[ ('correct',False,'correct'),('coverage',False,'coverage'),('citations',False,'citations'), ('clarity',2,'readability'),('critical_errors',['wrong_work'],'wrong_work')]) def test_each_answer_dimension_has_its_own_failure(tmp_path,monkeypatch,field,value,issue): case,answer,index,grade=fixture(tmp_path);grade[field]=value monkeypatch.setattr(e,'judge_answer',lambda *a:grade) score=e.evaluate(case,answer,index,tmp_path) assert not score['passed'] and issue in score['issues'] def test_judge_failure_is_not_a_success_or_removed_denominator(tmp_path,monkeypatch): case,answer,index,_=fixture(tmp_path) def unavailable(*a):raise RuntimeError('PRIVATE') monkeypatch.setattr(e,'judge_answer',unavailable) score=e.evaluate(case,answer,index,tmp_path) assert score['issues']==['judge_unavailable'] and not score['passed'] assert 'PRIVATE' not in str(score) def test_unretrieved_source_cannot_supply_a_required_value(tmp_path): case,answer,index,_=fixture(tmp_path) row={**index['E1'],'source_path':'other.md','source_url':'https://other.test'} result=e.evidence_coverage(case['evidence'],[row]) assert not result['all_required'] and result['recall']==0 def test_retrieval_and_answer_success_are_independent(tmp_path,monkeypatch): case,answer,index,grade=fixture(tmp_path);grade['correct']=False monkeypatch.setattr(e,'judge_answer',lambda *a:grade) score=e.evaluate(case,answer,index,tmp_path) assert score['retrieval']['selected_coverage']['all_required'] and not score['passed'] def test_negative_questions_do_not_count_missing_data_as_retrieval_errors(tmp_path,monkeypatch): case,answer,index,grade=fixture(tmp_path);case['expected_status']='no_answer';answer['status']='no_answer' monkeypatch.setattr(e,'judge_answer',lambda *a:grade) score=e.evaluate(case,answer,index,tmp_path) assert score['retrieval']['candidate_coverage'] is None def test_summary_uses_all_questions_and_exact_threshold_not_rounding(): rows=[{'id':str(i),'engine':engine,'category':'case','score':{'passed':i<75,'issues':[] if i<75 else ['false_refusal']}} for i in range(80) for engine in ['rag','graphrag']] score=e.summarize(rows) assert score['rag']['success_percent']==93.75 and not score['rag']['target_met'] assert score['paired']=={'questions':80,'both_pass':75,'rag_only':0,'graph_only':0} assert score['rag']['wilson_95_percent'][0]<93.75