313 lines
18 KiB
Python
313 lines
18 KiB
Python
"""Adversarial checks grade request fulfilment, not transport acceptance."""
|
|
import copy
|
|
import json
|
|
import pytest
|
|
from tools.answers import delivery_contract as c
|
|
from tools.answers.question_plan import QuestionPlan,PersonRequest,WorkRequest,Period,MeetingRequest
|
|
from tools.knowledge.sources import profile_units
|
|
|
|
|
|
def fixture():
|
|
units=profile_units({'title':'Eva Nová','path':'pages/students/2022/eva_nova/README.md',
|
|
'content':'# Diplomová práca 2027\nStretnutie 5.11.2026\nStav:\n- Funguje model BERT.\n'
|
|
'Stretnutie 1.10.2026\nÚlohy:\n- Vyhodnoťte model.'},include_records=True)
|
|
row=next(r for r in units if 'Funguje' in r['text'])
|
|
plan=QuestionPlan(question='Aké výsledky má Eva Nová pri diplomovej práci 2027?',intent='facets',
|
|
people=[PersonRequest(name='Eva Nová')],works=[WorkRequest(person='Eva Nová',work_type='diploma_thesis',years=[2027])],facets=['results'])
|
|
result={'status':'ok','accepted_for_delivery':True,'index_version':{'version':'v'},
|
|
'answer':'Eva Nová: funguje model BERT.\nZdroj: '+row['source_url'],
|
|
'evidence':[row],'claims':[],'source_urls':[row['source_url']],'presentation_notes':[]}
|
|
return plan,result,{'index_version':{'version':'v'},'units':units}
|
|
|
|
|
|
def audit(**changes):
|
|
def model(messages):
|
|
p=json.loads(messages[-1]['content'])
|
|
value={'plan_matches_request':True,'all_requirements_identified':True,'facts_supported':True,'clear':True,
|
|
'requirements':[{'description':p['resolved_question'],'question_segments':['Q1'],'status':'answered',
|
|
'answer_segments':[p['answer_segments'][0]['id']],'evidence_ids':['E1']}]}
|
|
value.update(changes)
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
return model
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
def test_matching_request_plan_original_evidence_and_final_answer_pass(engine):
|
|
plan,result,index=fixture()
|
|
out=c.enforce(plan.question,engine,plan,result,index=index,complete=audit())
|
|
assert out['accepted_for_delivery'] and out['delivery_check']['code']=='complete'
|
|
assert out['delivery_check']['request_fingerprint']==c.fingerprint(plan.question,engine)
|
|
|
|
|
|
@pytest.mark.parametrize('change,code',[
|
|
({'person':'Dana Iná'},'wrong_person'),({'work_year':2024},'wrong_work_or_meeting'),
|
|
({'phase':'planned'},'altered_evidence'),({'text':'Dokončila celú prácu.'},'altered_evidence')])
|
|
def test_true_but_wrong_or_relabelled_evidence_cannot_be_waived_by_friendly_model(change,code):
|
|
plan,result,index=fixture();result=copy.deepcopy(result);result['evidence'][0].update(change)
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit())
|
|
assert not out['accepted_for_delivery'] and out['delivery_check']['code']==code
|
|
assert 'Dokončila' not in out['answer'] and 'evidence' not in out
|
|
|
|
|
|
@pytest.mark.parametrize('field',['record_date','event_date','target_date'])
|
|
def test_temporal_range_is_checked_even_outside_meeting_route(field):
|
|
plan,result,index=fixture();plan.period=Period(start='2026-10-01',end='2026-10-01',field=field)
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit())
|
|
assert out['delivery_check']['code']=='wrong_period'
|
|
|
|
|
|
def test_latest_is_verified_against_published_records_not_only_the_selected_old_record():
|
|
plan,result,index=fixture();plan.meeting=MeetingRequest(selection='latest')
|
|
result['evidence']=[next(r for r in index['units'] if 'Vyhodnoťte' in r['text'])]
|
|
assert c.enforce(plan.question,'graphrag',plan,result,index=index,complete=audit())['delivery_check']['code']=='wrong_work_or_meeting'
|
|
|
|
|
|
def test_missing_person_is_not_silently_removed_from_comparison():
|
|
plan,result,index=fixture();plan.people.append(PersonRequest(name='Dana Iná'))
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit())
|
|
assert out['delivery_check']['code']=='missing_person'
|
|
|
|
|
|
@pytest.mark.parametrize('change',[
|
|
{'facts_supported':False},{'plan_matches_request':False},
|
|
{'requirements':[{'description':'results','question_segments':['Q1'],'status':'answered','answer_segments':['invented'],'evidence_ids':['fake']}]}])
|
|
def test_delivery_is_not_success_if_request_or_final_text_fails_review(change):
|
|
plan,result,index=fixture()
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(**change))
|
|
assert not out['accepted_for_delivery'] and out['reason']=='answer_rejected'
|
|
assert 'evidence' not in out and 'synthesis' not in out
|
|
|
|
|
|
def test_explicit_missing_person_is_partial_only_when_missing_notice_is_really_in_answer():
|
|
plan,result,index=fixture();plan.question+=' A Dana Iná?';plan.people.append(PersonRequest(name='Dana Iná'))
|
|
notice='Dana Iná: požadované podklady sa nenašli.'
|
|
result['presentation_notes']=[notice];result['answer']+='\n\n'+notice
|
|
model=audit(requirements=[{'description':'Dana Iná','question_segments':['Q1'],'status':'missing_disclosed','answer_segments':['A2'],'evidence_ids':[]},
|
|
{'description':'Eva Nová','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}])
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model)
|
|
assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial'
|
|
|
|
|
|
def test_distributed_requirement_uses_question_references_without_manufacturing_a_quote():
|
|
plan,result,index=fixture()
|
|
model=audit(requirements=[{'description':'Results of the specified person and work',
|
|
'question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}])
|
|
assert c.enforce(plan.question,'rag',plan,result,index=index,complete=model)['accepted_for_delivery']
|
|
|
|
|
|
def test_unknown_question_segment_cannot_pass_review():
|
|
plan,result,index=fixture()
|
|
model=audit(requirements=[{'description':'Results','question_segments':['Q999'],
|
|
'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}])
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model)
|
|
assert out['delivery_check']['code']=='invalid_review'
|
|
|
|
|
|
def test_negative_semantic_verdict_is_never_retried_to_get_acceptance():
|
|
plan,result,index=fixture();calls=[]
|
|
def model(messages):
|
|
calls.append(1)
|
|
return audit(facts_supported=False)(messages)
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model)
|
|
assert out['delivery_check']['code']=='request_not_fulfilled' and len(calls)==1
|
|
|
|
|
|
def test_verification_outage_is_distinct_and_diagnostic_does_not_leak_exception():
|
|
plan,result,index=fixture()
|
|
def down(_):raise RuntimeError('SECRET_API_KEY_OR_PROVIDER_RESPONSE')
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=down)
|
|
assert out['reason']=='verification_unavailable' and out['delivery_check']['layer']=='verification'
|
|
assert 'SECRET' not in json.dumps(out)
|
|
|
|
|
|
def test_self_declared_foreign_citation_does_not_authorize_itself():
|
|
plan,result,index=fixture();result['source_urls'].append('https://invented.test')
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit())
|
|
assert out['delivery_check']['code']=='invalid_citation'
|
|
|
|
|
|
def test_metadata_work_filters_are_checked_against_actual_execution_plan():
|
|
plan,result,index=fixture();plan.intent='metadata';result['evidence']=[]
|
|
result['claims']=[{'person':'Eva Nová','work_type':'bachelor_thesis','year':2024}]
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=lambda _:pytest.fail('Exact metadata does not need prose review'))
|
|
assert out['delivery_check']['code']=='wrong_work_or_meeting'
|
|
|
|
|
|
@pytest.mark.parametrize('change',[{'all_requirements_identified':False},{'clear':False}])
|
|
def test_reviewer_doubt_delivers_supported_facts_as_partial_with_notice(change):
|
|
# An unsupported fact or a different request is still rejected (above). A
|
|
# reviewer's doubt about completeness keeps the checked facts, disclosed.
|
|
plan,result,index=fixture()
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(**change))
|
|
assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial'
|
|
assert c.UNCERTAIN_NOTICE in out['answer']
|
|
|
|
|
|
def test_all_requirements_unfulfilled_is_still_rejected():
|
|
plan,result,index=fixture()
|
|
model=audit(requirements=[{'description':'results','question_segments':['Q1'],'status':'unfulfilled','answer_segments':[],'evidence_ids':[]}])
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model)
|
|
assert not out['accepted_for_delivery'] and out['reason']=='answer_rejected'
|
|
|
|
|
|
def test_spent_budget_delivers_checked_facts_with_an_explicit_unverified_notice():
|
|
from tools.answers import model_client as mc
|
|
plan,result,index=fixture();called=[]
|
|
token=mc.start_deadline(3)
|
|
try:
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=lambda m:called.append(m))
|
|
finally:
|
|
mc.reset_deadline(token)
|
|
assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial'
|
|
assert c.UNVERIFIED_NOTICE in out['answer'] and not called
|
|
|
|
|
|
def test_requirement_met_by_a_backend_notice_needs_no_source_evidence():
|
|
plan,result,index=fixture()
|
|
notice='Zobrazené sú príklady z obmedzeného výberu.'
|
|
result['presentation_notes']=[notice];result['answer']+='\n\n'+notice
|
|
model=audit(requirements=[
|
|
{'description':'answer','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']},
|
|
{'description':'disclose limited list','question_segments':['Q1'],'status':'answered','answer_segments':['A2'],'evidence_ids':[]}])
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model)
|
|
assert out['accepted_for_delivery'] and out['delivery_check']['code']=='complete'
|
|
|
|
|
|
def test_answered_requirement_without_evidence_is_partial_unless_all_lack_it():
|
|
plan,result,index=fixture()
|
|
one=audit(requirements=[
|
|
{'description':'answer','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']},
|
|
{'description':'extra','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':[]}])
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=one)
|
|
assert out['accepted_for_delivery'] and c.UNCERTAIN_NOTICE in out['answer']
|
|
none=audit(requirements=[{'description':'answer','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':[]}])
|
|
assert not c.enforce(plan.question,'rag',plan,result,index=index,complete=none)['accepted_for_delivery']
|
|
|
|
|
|
def _prose_fixture():
|
|
plan,result,index=fixture()
|
|
url=result['source_urls'][0]
|
|
plain='- Diplomová práca 2027; zaznamenaný výsledok: Funguje model BERT.'
|
|
readable='Diplomová práca 2027: Funguje jej model BERT.'
|
|
result['answer']='**Eva Nová**\n\n'+readable+'\n\nZdroje: '+url
|
|
return plan,result,index,[[readable,plain]],readable,plain
|
|
|
|
|
|
def test_audited_readable_wording_is_delivered():
|
|
plan,result,index,swaps,readable,plain=_prose_fixture()
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(),prose_swaps=swaps)
|
|
assert out['delivery_check']['code']=='complete' and readable in out['answer'] and plain not in out['answer']
|
|
|
|
|
|
def test_rejected_wording_falls_back_to_verbatim_lines_with_their_own_audit():
|
|
plan,result,index,swaps,readable,plain=_prose_fixture();calls=[]
|
|
def model(messages):
|
|
p=json.loads(messages[-1]['content'])
|
|
wording=any(readable in s['text'] for s in p['answer_segments'])
|
|
calls.append(wording)
|
|
return audit(facts_supported=not wording)(messages)
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model,prose_swaps=swaps)
|
|
assert calls==[True,False]
|
|
assert out['accepted_for_delivery'] and plain in out['answer'] and readable not in out['answer']
|
|
|
|
|
|
def test_unaudited_wording_is_never_delivered():
|
|
from tools.answers import model_client as mc
|
|
plan,result,index,swaps,readable,plain=_prose_fixture()
|
|
token=mc.start_deadline(3)
|
|
try:
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=lambda m:pytest.fail('no time to audit'),prose_swaps=swaps)
|
|
finally:
|
|
mc.reset_deadline(token)
|
|
assert out['delivery_check']['code']=='partial' and c.UNVERIFIED_NOTICE in out['answer']
|
|
assert plain in out['answer'] and readable not in out['answer']
|
|
|
|
|
|
def test_wording_fallback_never_rescues_a_different_request():
|
|
plan,result,index,swaps,readable,plain=_prose_fixture()
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(plan_matches_request=False),prose_swaps=swaps)
|
|
assert not out['accepted_for_delivery'] and out['reason']=='answer_rejected'
|
|
|
|
|
|
def test_audit_evidence_is_grouped_without_losing_attribution():
|
|
from tools.answers.delivery_contract import audit_evidence
|
|
rows={
|
|
'a':{'id':'a','person':'Eva Nová','work_id':'w1','work_label':'Diplomová práca 2027','work_type':'diplomova',
|
|
'work_year':2027,'meeting_id':'m-long-internal-id','phase':'result','text':'Hotový model.',
|
|
'record_date':'2026-10-01','record_date_label':'2026-10-01'},
|
|
'b':{'id':'b','person':'Eva Nová','work_id':'w1','work_label':'Diplomová práca 2027','work_type':'diplomova',
|
|
'work_year':2027,'phase':'planned','text':'Vyhodnoťte model.'},
|
|
'c':{'id':'c','person':'Adam Nový','work_id':'w2','work_label':'Bakalárska práca 2025','work_type':'bakalarska',
|
|
'work_year':2025,'phase':'progress','text':'Zbiera dáta.','event_date':'2024-11-02'},
|
|
}
|
|
groups=audit_evidence(rows,{'a':'E1','b':'E2','c':'E3'})
|
|
assert [(g['person'],g['work_label'],[u['id'] for u in g['evidence']]) for g in groups]==[
|
|
('Eva Nová','Diplomová práca 2027',['E1','E2']),('Adam Nový','Bakalárska práca 2025',['E3'])]
|
|
assert groups[0]['evidence'][0]=={'id':'E1','phase':'result','record_date':'2026-10-01','text':'Hotový model.'}
|
|
assert groups[0]['evidence'][1]=={'id':'E2','phase':'planned','text':'Vyhodnoťte model.'}
|
|
assert groups[1]['evidence'][0]['event_date']=='2024-11-02'
|
|
assert 'meeting_id' not in str(groups)
|
|
|
|
|
|
def test_verbatim_quoted_unit_is_sent_once_as_a_reference():
|
|
from tools.answers.delivery_contract import audit_evidence
|
|
text='- Dotrénované 4 modely, slovakt5-base, slavict5-base, mt5-base, umt5-base.'
|
|
rows={'a':{'id':'a','person':'Eva Nová','phase':'result','text':text},
|
|
'b':{'id':'b','person':'Eva Nová','phase':'result','text':'- Model s augmentovanými dátami je o dosť lepší než pôvodný.'}}
|
|
segments={'A1':'**Eva Nová**','A2':'- Bakalárska práca 2025; zaznamenaný výsledok: '+text[2:],
|
|
'A3':'Bakalárska práca 2025: Zlepšila model pomocou augmentovaných dát.'}
|
|
units=audit_evidence(rows,{'a':'E1','b':'E2'},segments)[0]['evidence']
|
|
assert units[0]=={'id':'E1','phase':'result','text_quoted_verbatim_in':'A2'}
|
|
# Paraphrased wording must still be compared with the source text.
|
|
assert units[1]['text']==rows['b']['text']
|
|
|
|
|
|
def test_discovery_audit_receives_the_executed_decomposition():
|
|
from tools.answers import delivery_contract as dc
|
|
plan,result,index=fixture()
|
|
plan=plan.model_copy(update={'intent':'discovery','question':'Ktorí študenti robili s LLM a čo im fungovalo?'})
|
|
seen=[]
|
|
def review(messages):
|
|
seen.append(json.loads(messages[-1]['content']))
|
|
raise RuntimeError('stop')
|
|
with pytest.raises(dc.ContractError):
|
|
dc.semantic_check(plan.question,plan,result,complete=review)
|
|
assert seen[0]['proposed_requirements']==[{'id':'R1','text':'Ktorí študenti robili s LLM'},
|
|
{'id':'R2','text':'čo im fungovalo'}]
|
|
|
|
|
|
def test_stray_reference_in_a_long_list_is_dropped_not_used():
|
|
plan,result,index=fixture()
|
|
model=audit(requirements=[{'description':'Results','question_segments':['Q1'],
|
|
'status':'answered','answer_segments':['A1'],'evidence_ids':['E1','A1']}])
|
|
assert c.enforce(plan.question,'rag',plan,result,index=index,complete=model)['accepted_for_delivery']
|
|
|
|
|
|
def test_answered_requirement_citing_only_unknown_units_has_no_support():
|
|
plan,result,index=fixture()
|
|
model=audit(requirements=[{'description':'Results','question_segments':['Q1'],
|
|
'status':'answered','answer_segments':['A1'],'evidence_ids':['E999']}])
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model)
|
|
assert out['delivery_check']['code']=='request_not_fulfilled'
|
|
|
|
|
|
def test_doubt_disclosed_by_the_backend_notice_is_delivered_as_partial():
|
|
plan,result,index=fixture()
|
|
note='Profil obsahuje aj záznamy bez úplného dátumu; nemožno spoľahlivo určiť, či sú novšie.'
|
|
result={**result,'answer':result['answer']+'\n\n'+note,'presentation_notes':[note]}
|
|
def doubt(segment):
|
|
def model(messages):
|
|
p=json.loads(messages[-1]['content'])
|
|
ids=[s['id'] for s in p['answer_segments'] if segment(s['text'])]
|
|
value={'plan_matches_request':True,'all_requirements_identified':True,'facts_supported':True,'clear':True,
|
|
'requirements':[{'description':'Latest meeting','question_segments':['Q1'],'status':'uncertain',
|
|
'answer_segments':ids,'evidence_ids':['E1']}]}
|
|
return {'choices':[{'message':{'content':json.dumps(value)}}]}
|
|
return model
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=doubt(lambda t:True))
|
|
assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial'
|
|
assert c.UNCERTAIN_NOTICE in out['answer']
|
|
# The same doubt without the backend's own notice is still rejected.
|
|
out=c.enforce(plan.question,'rag',plan,result,index=index,complete=doubt(lambda t:note not in t))
|
|
assert not out['accepted_for_delivery']
|