185 lines
10 KiB
Python
185 lines
10 KiB
Python
"""An evaluator must detect mistakes, not reward a backend's acceptance flag."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import pytest
|
|
|
|
from tools.evaluation import user_questions as e
|
|
|
|
|
|
def fixture(tmp_path):
|
|
path=tmp_path/'pages/students/2020/eva/README.md';path.parent.mkdir(parents=True)
|
|
path.write_text('---\ntitle: Eva Nová \npublished: true\n---\n# Diplomová práca 2026\nStav:\n- Implementovaný model BERT.\n')
|
|
gold={'path':str(path.relative_to(tmp_path)),'sha256':e.digest(path),'url':'https://example.test/eva',
|
|
'start':7,'end':7,'quote':'- Implementovaný model BERT.',
|
|
'requirements':['BERT implementation, not graduation.'],'retrieval_terms':[['BERT']]}
|
|
case={'id':'T001','category':'result','question':'Čo vytvorila Eva Nová?',
|
|
'expected_status':'ok','evidence':[gold],'response_requirements':[]}
|
|
row={'id':'E1','person':'Eva Nová','source_path':gold['path'],'source_url':gold['url'],
|
|
'line':7,'end_line':7,'text':gold['quote'],'work_label':'Diplomová práca 2026'}
|
|
answer={'status':'ok','accepted_for_delivery':True,'answer':'Implementovala model BERT.\nZdroj: '+gold['url'],
|
|
'source_urls':[gold['url']],'evidence':[row],
|
|
'retrieval_diagnostics':{'candidate_ids':['E1'],'strategy':'content_checkpoint'}}
|
|
grade={'correct':True,'coverage':True,'citations':True,'appropriate_response':True,'clarity':4,'critical_errors':[],'reasons':[]}
|
|
return case,answer,{'E1':row},grade
|
|
|
|
|
|
def test_gold_is_verified_against_original_bytes_and_lines(tmp_path):
|
|
case,_,_,_=fixture(tmp_path)
|
|
data={'cases':[case]}
|
|
assert not e.audit_dataset(data,tmp_path)
|
|
case['evidence'][0]['quote']='Made up text'
|
|
assert ('T001','wrong_source_lines') in e.audit_dataset(data,tmp_path)
|
|
|
|
|
|
def test_gold_duplicate_or_changed_source_is_never_silently_accepted(tmp_path):
|
|
case,_,_,_=fixture(tmp_path)
|
|
duplicate=copy.deepcopy(case);duplicate['id']='other'
|
|
case['evidence'][0]['sha256']='old'
|
|
issues=e.audit_dataset({'cases':[case,duplicate]},tmp_path)
|
|
assert ('T001','source_changed') in issues and ('other','duplicate_case') in issues
|
|
|
|
|
|
def test_correct_source_owner_whitespace_is_not_a_false_failure(tmp_path,monkeypatch):
|
|
case,answer,index,grade=fixture(tmp_path)
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert score['passed'] and score['retrieval']['candidate_coverage']['all_required']
|
|
assert score['retrieval']['selected_coverage']['all_required']
|
|
|
|
|
|
@pytest.mark.parametrize('change,issue',[
|
|
({'answer':'[x](/made-up)'},'unsupported_citation'),
|
|
({'source_urls':[]},'missing_citation'),
|
|
({'status':'no_answer'},'false_refusal'),
|
|
({'accepted_for_delivery':False},'delivery_rejected'),
|
|
])
|
|
def test_acceptance_or_a_friendly_judge_cannot_waive_mechanical_errors(tmp_path,monkeypatch,change,issue):
|
|
case,answer,index,grade=fixture(tmp_path);answer.update(change)
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert not score['passed'] and issue in score['issues']
|
|
|
|
|
|
@pytest.mark.parametrize('field,value,issue',[
|
|
('correct',False,'correct'),('coverage',False,'coverage'),('citations',False,'citations'),
|
|
('clarity',2,'readability'),('critical_errors',['wrong_work'],'wrong_work')])
|
|
def test_each_answer_dimension_has_its_own_failure(tmp_path,monkeypatch,field,value,issue):
|
|
case,answer,index,grade=fixture(tmp_path);grade[field]=value
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert not score['passed'] and issue in score['issues']
|
|
|
|
|
|
def test_judge_failure_is_not_a_success_or_removed_denominator(tmp_path,monkeypatch):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
def unavailable(*a):raise RuntimeError('PRIVATE')
|
|
monkeypatch.setattr(e,'judge_answer',unavailable)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert score['issues']==['judge_unavailable'] and not score['passed']
|
|
assert 'PRIVATE' not in str(score)
|
|
|
|
|
|
def test_unretrieved_source_cannot_supply_a_required_value(tmp_path):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
row={**index['E1'],'source_path':'other.md','source_url':'https://other.test'}
|
|
result=e.evidence_coverage(case['evidence'],[row])
|
|
assert not result['all_required'] and result['recall']==0
|
|
|
|
|
|
def test_retrieval_and_answer_success_are_independent(tmp_path,monkeypatch):
|
|
case,answer,index,grade=fixture(tmp_path);grade['correct']=False
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert score['retrieval']['selected_coverage']['all_required'] and not score['passed']
|
|
|
|
|
|
def test_negative_questions_do_not_count_missing_data_as_retrieval_errors(tmp_path,monkeypatch):
|
|
case,answer,index,grade=fixture(tmp_path);case['expected_status']='no_answer';answer['status']='no_answer'
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert score['retrieval']['candidate_coverage'] is None
|
|
|
|
|
|
def test_summary_uses_all_questions_and_exact_threshold_not_rounding():
|
|
rows=[{'id':str(i),'engine':engine,'category':'case','score':{'passed':i<75,'issues':[] if i<75 else ['false_refusal']}}
|
|
for i in range(80) for engine in ['rag','graphrag']]
|
|
score=e.summarize(rows)
|
|
assert score['rag']['success_percent']==93.75 and not score['rag']['target_met']
|
|
assert score['paired']=={'questions':80,'both_pass':75,'rag_only':0,'graph_only':0}
|
|
assert score['rag']['wilson_95_percent'][0]<93.75<score['rag']['wilson_95_percent'][1]
|
|
|
|
|
|
def test_brief_overview_budget_is_a_real_acceptance_criterion(tmp_path,monkeypatch):
|
|
case,answer,index,grade=fixture(tmp_path);case['max_words']=5;answer['answer']='word '*10
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
score=e.evaluate(case,answer,index,tmp_path)
|
|
assert 'verbosity_budget' in score['issues']
|
|
|
|
|
|
def test_raw_windows_keep_backlog_boundaries_instead_of_trusting_annotation(tmp_path):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
path=tmp_path/case['evidence'][0]['path']
|
|
path.write_text(path.read_text()+'Stretnutie 5.5.2026\nStav:\n- Funguje systém.\nZásobník úloh:\n- Vyhodnoťte model.\n')
|
|
answer['evidence']=[{**index['E1'],'line':12,'end_line':12,'text':'- Vyhodnoťte model.'}]
|
|
text=e.source_windows(case,answer,tmp_path)[0]['lines']
|
|
assert 'Zásobník úloh:' in text and 'Stretnutie 5.5.2026' in text
|
|
|
|
|
|
def test_literal_links_inside_delivered_evidence_are_provided_sources(tmp_path,monkeypatch):
|
|
case,answer,index,grade=fixture(tmp_path)
|
|
url='https://example.test/technical-reference'
|
|
text=index['E1']['text']+' '+url
|
|
path=tmp_path/case['evidence'][0]['path'];path.write_text(path.read_text().replace(index['E1']['text'],text))
|
|
index['E1']['text']=text;answer['answer']+='\n'+url
|
|
monkeypatch.setattr(e,'judge_answer',lambda *a:grade)
|
|
assert 'unsupported_citation' not in e.evaluate(case,answer,index,tmp_path)['issues']
|
|
|
|
|
|
def test_nearest_work_and_section_are_explicit_even_with_nested_markdown(tmp_path):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
path=tmp_path/case['evidence'][0]['path']
|
|
path.write_text('---\ntitle: Eva Nová\n---\n# Diplomová práca 2027\nCiele:\n- Vytvorte systém.\n## Bakalárska práca 2024\nStretnutie 5.5.2026\nStav:\n- Implementovaný BERT.\n')
|
|
case['evidence']=[];answer['evidence']=[{**index['E1'],'line':10,'end_line':10}]
|
|
text=e.source_windows(case,answer,tmp_path)[0]['lines']
|
|
assert '7: Bakalárska práca 2024' in text and '9: Stav:' in text
|
|
assert 'Original context for source lines 10' in text
|
|
|
|
|
|
@pytest.mark.parametrize('text',['Model zatiaľ nefunguje.','Nepasuje konfigurácia modelu.',
|
|
'Použitá konfigurácia je nekompatibilná.','Výsledky je ťažké porovnať pomocou BLEU.'])
|
|
def test_technical_difficulty_does_not_require_the_literal_word_problem(text):
|
|
from tools.answers.content_selection import facets_for
|
|
assert 'problems' in facets_for(text,'progress','Aké problémy riešil študent?','progress')
|
|
|
|
|
|
@pytest.mark.parametrize('text',['Dátum záznamu je 5.5.2026.',
|
|
'Diplomový projekt 2026; priebežný stav; záznam 2026-05-05'])
|
|
def test_matching_record_date_is_not_evidence_of_an_unsupported_time_error(tmp_path,text):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
path=tmp_path/case['evidence'][0]['path'];path.write_text(path.read_text()+'Stretnutie 5.5.2026\n')
|
|
grade=e.Judge(correct=False,coverage=True,citations=True,appropriate_response=True,clarity=4,
|
|
critical_errors=['unsupported_time'],reasons=['unsupported_time'],proofs=[e.JudgeProof(
|
|
code='unsupported_time',answer_quote=text,source_path=case['evidence'][0]['path'],source_quote='Stretnutie 5.5.2026')])
|
|
with pytest.raises(ValueError,match='explicitly matching'):
|
|
e.validate_judge_proofs(grade,text,{case['evidence'][0]['path']},tmp_path)
|
|
|
|
|
|
def test_a_year_inferred_from_a_work_is_still_a_valid_critical_error(tmp_path):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
path=tmp_path/case['evidence'][0]['path'];path.write_text(path.read_text()+'Stretnutie 5.5.\n')
|
|
text='Dátum záznamu je 5.5.2026.'
|
|
grade=e.Judge(correct=False,coverage=True,citations=True,appropriate_response=True,clarity=4,
|
|
critical_errors=['unsupported_time'],reasons=['unsupported_time'],proofs=[e.JudgeProof(
|
|
code='unsupported_time',answer_quote=text,source_path=case['evidence'][0]['path'],source_quote='Stretnutie 5.5.')])
|
|
e.validate_judge_proofs(grade,text,{case['evidence'][0]['path']},tmp_path)
|
|
|
|
|
|
def test_virtual_meeting_dates_are_kept_but_task_headings_cannot_replace_them(tmp_path):
|
|
case,answer,index,_=fixture(tmp_path)
|
|
path=tmp_path/case['evidence'][0]['path']
|
|
path.write_text('---\ntitle: Eva Nová\n---\n# Diplomová práca 2026\nVirtuálne stretnutie 13.11:2020:\nStav:\n- Funguje model.\nÚlohy na ďalšie stretnutie:\n\n\n- Pripravte experiment.\n')
|
|
case['evidence']=[];answer['evidence']=[{**index['E1'],'line':11,'end_line':11}]
|
|
text=e.source_windows(case,answer,tmp_path)[0]['lines']
|
|
assert 'Virtuálne stretnutie 13.11:2020:' in text
|