"""Adversarial checks grade request fulfilment, not transport acceptance.""" import copy import json import pytest from tools.answers import delivery_contract as c from tools.answers.question_plan import QuestionPlan,PersonRequest,WorkRequest,Period,MeetingRequest from tools.knowledge.sources import profile_units def fixture(): units=profile_units({'title':'Eva Nová','path':'pages/students/2022/eva_nova/README.md', 'content':'# Diplomová práca 2027\nStretnutie 5.11.2026\nStav:\n- Funguje model BERT.\n' 'Stretnutie 1.10.2026\nÚlohy:\n- Vyhodnoťte model.'},include_records=True) row=next(r for r in units if 'Funguje' in r['text']) plan=QuestionPlan(question='Aké výsledky má Eva Nová pri diplomovej práci 2027?',intent='facets', people=[PersonRequest(name='Eva Nová')],works=[WorkRequest(person='Eva Nová',work_type='diploma_thesis',years=[2027])],facets=['results']) result={'status':'ok','accepted_for_delivery':True,'index_version':{'version':'v'}, 'answer':'Eva Nová: funguje model BERT.\nZdroj: '+row['source_url'], 'evidence':[row],'claims':[],'source_urls':[row['source_url']],'presentation_notes':[]} return plan,result,{'index_version':{'version':'v'},'units':units} def audit(**changes): def model(messages): p=json.loads(messages[-1]['content']) value={'plan_matches_request':True,'all_requirements_identified':True,'facts_supported':True,'clear':True, 'requirements':[{'description':p['resolved_question'],'question_segments':['Q1'],'status':'answered', 'answer_segments':[p['answer_segments'][0]['id']],'evidence_ids':['E1']}]} value.update(changes) return {'choices':[{'message':{'content':json.dumps(value)}}]} return model @pytest.mark.parametrize('engine',['rag','graphrag']) def test_matching_request_plan_original_evidence_and_final_answer_pass(engine): plan,result,index=fixture() out=c.enforce(plan.question,engine,plan,result,index=index,complete=audit()) assert out['accepted_for_delivery'] and out['delivery_check']['code']=='complete' assert out['delivery_check']['request_fingerprint']==c.fingerprint(plan.question,engine) @pytest.mark.parametrize('change,code',[ ({'person':'Dana Iná'},'wrong_person'),({'work_year':2024},'wrong_work_or_meeting'), ({'phase':'planned'},'altered_evidence'),({'text':'Dokončila celú prácu.'},'altered_evidence')]) def test_true_but_wrong_or_relabelled_evidence_cannot_be_waived_by_friendly_model(change,code): plan,result,index=fixture();result=copy.deepcopy(result);result['evidence'][0].update(change) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit()) assert not out['accepted_for_delivery'] and out['delivery_check']['code']==code assert 'Dokončila' not in out['answer'] and 'evidence' not in out @pytest.mark.parametrize('field',['record_date','event_date','target_date']) def test_temporal_range_is_checked_even_outside_meeting_route(field): plan,result,index=fixture();plan.period=Period(start='2026-10-01',end='2026-10-01',field=field) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit()) assert out['delivery_check']['code']=='wrong_period' def test_latest_is_verified_against_published_records_not_only_the_selected_old_record(): plan,result,index=fixture();plan.meeting=MeetingRequest(selection='latest') result['evidence']=[next(r for r in index['units'] if 'Vyhodnoťte' in r['text'])] assert c.enforce(plan.question,'graphrag',plan,result,index=index,complete=audit())['delivery_check']['code']=='wrong_work_or_meeting' def test_missing_person_is_not_silently_removed_from_comparison(): plan,result,index=fixture();plan.people.append(PersonRequest(name='Dana Iná')) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit()) assert out['delivery_check']['code']=='missing_person' @pytest.mark.parametrize('change',[ {'facts_supported':False},{'plan_matches_request':False}, {'requirements':[{'description':'results','question_segments':['Q1'],'status':'answered','answer_segments':['invented'],'evidence_ids':['fake']}]}]) def test_delivery_is_not_success_if_request_or_final_text_fails_review(change): plan,result,index=fixture() out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(**change)) assert not out['accepted_for_delivery'] and out['reason']=='answer_rejected' assert 'evidence' not in out and 'synthesis' not in out def test_explicit_missing_person_is_partial_only_when_missing_notice_is_really_in_answer(): plan,result,index=fixture();plan.question+=' A Dana Iná?';plan.people.append(PersonRequest(name='Dana Iná')) notice='Dana Iná: požadované podklady sa nenašli.' result['presentation_notes']=[notice];result['answer']+='\n\n'+notice model=audit(requirements=[{'description':'Dana Iná','question_segments':['Q1'],'status':'missing_disclosed','answer_segments':['A2'],'evidence_ids':[]}, {'description':'Eva Nová','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}]) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model) assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial' def test_distributed_requirement_uses_question_references_without_manufacturing_a_quote(): plan,result,index=fixture() model=audit(requirements=[{'description':'Results of the specified person and work', 'question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}]) assert c.enforce(plan.question,'rag',plan,result,index=index,complete=model)['accepted_for_delivery'] def test_unknown_question_segment_cannot_pass_review(): plan,result,index=fixture() model=audit(requirements=[{'description':'Results','question_segments':['Q999'], 'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}]) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model) assert out['delivery_check']['code']=='invalid_review' def test_negative_semantic_verdict_is_never_retried_to_get_acceptance(): plan,result,index=fixture();calls=[] def model(messages): calls.append(1) return audit(facts_supported=False)(messages) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model) assert out['delivery_check']['code']=='request_not_fulfilled' and len(calls)==1 def test_verification_outage_is_distinct_and_diagnostic_does_not_leak_exception(): plan,result,index=fixture() def down(_):raise RuntimeError('SECRET_API_KEY_OR_PROVIDER_RESPONSE') out=c.enforce(plan.question,'rag',plan,result,index=index,complete=down) assert out['reason']=='verification_unavailable' and out['delivery_check']['layer']=='verification' assert 'SECRET' not in json.dumps(out) def test_self_declared_foreign_citation_does_not_authorize_itself(): plan,result,index=fixture();result['source_urls'].append('https://invented.test') out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit()) assert out['delivery_check']['code']=='invalid_citation' def test_metadata_work_filters_are_checked_against_actual_execution_plan(): plan,result,index=fixture();plan.intent='metadata';result['evidence']=[] result['claims']=[{'person':'Eva Nová','work_type':'bachelor_thesis','year':2024}] out=c.enforce(plan.question,'rag',plan,result,index=index,complete=lambda _:pytest.fail('Exact metadata does not need prose review')) assert out['delivery_check']['code']=='wrong_work_or_meeting' @pytest.mark.parametrize('change',[{'all_requirements_identified':False},{'clear':False}]) def test_reviewer_doubt_delivers_supported_facts_as_partial_with_notice(change): # An unsupported fact or a different request is still rejected (above). A # reviewer's doubt about completeness keeps the checked facts, disclosed. plan,result,index=fixture() out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(**change)) assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial' assert c.UNCERTAIN_NOTICE in out['answer'] def test_all_requirements_unfulfilled_is_still_rejected(): plan,result,index=fixture() model=audit(requirements=[{'description':'results','question_segments':['Q1'],'status':'unfulfilled','answer_segments':[],'evidence_ids':[]}]) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model) assert not out['accepted_for_delivery'] and out['reason']=='answer_rejected' def test_spent_budget_delivers_checked_facts_with_an_explicit_unverified_notice(): from tools.answers import model_client as mc plan,result,index=fixture();called=[] token=mc.start_deadline(3) try: out=c.enforce(plan.question,'rag',plan,result,index=index,complete=lambda m:called.append(m)) finally: mc.reset_deadline(token) assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial' assert c.UNVERIFIED_NOTICE in out['answer'] and not called def test_requirement_met_by_a_backend_notice_needs_no_source_evidence(): plan,result,index=fixture() notice='Zobrazené sú príklady z obmedzeného výberu.' result['presentation_notes']=[notice];result['answer']+='\n\n'+notice model=audit(requirements=[ {'description':'answer','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}, {'description':'disclose limited list','question_segments':['Q1'],'status':'answered','answer_segments':['A2'],'evidence_ids':[]}]) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model) assert out['accepted_for_delivery'] and out['delivery_check']['code']=='complete' def test_answered_requirement_without_evidence_is_partial_unless_all_lack_it(): plan,result,index=fixture() one=audit(requirements=[ {'description':'answer','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':['E1']}, {'description':'extra','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':[]}]) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=one) assert out['accepted_for_delivery'] and c.UNCERTAIN_NOTICE in out['answer'] none=audit(requirements=[{'description':'answer','question_segments':['Q1'],'status':'answered','answer_segments':['A1'],'evidence_ids':[]}]) assert not c.enforce(plan.question,'rag',plan,result,index=index,complete=none)['accepted_for_delivery'] def _prose_fixture(): plan,result,index=fixture() url=result['source_urls'][0] plain='- Diplomová práca 2027; zaznamenaný výsledok: Funguje model BERT.' readable='Diplomová práca 2027: Funguje jej model BERT.' result['answer']='**Eva Nová**\n\n'+readable+'\n\nZdroje: '+url return plan,result,index,[[readable,plain]],readable,plain def test_audited_readable_wording_is_delivered(): plan,result,index,swaps,readable,plain=_prose_fixture() out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(),prose_swaps=swaps) assert out['delivery_check']['code']=='complete' and readable in out['answer'] and plain not in out['answer'] def test_rejected_wording_falls_back_to_verbatim_lines_with_their_own_audit(): plan,result,index,swaps,readable,plain=_prose_fixture();calls=[] def model(messages): p=json.loads(messages[-1]['content']) wording=any(readable in s['text'] for s in p['answer_segments']) calls.append(wording) return audit(facts_supported=not wording)(messages) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model,prose_swaps=swaps) assert calls==[True,False] assert out['accepted_for_delivery'] and plain in out['answer'] and readable not in out['answer'] def test_unaudited_wording_is_never_delivered(): from tools.answers import model_client as mc plan,result,index,swaps,readable,plain=_prose_fixture() token=mc.start_deadline(3) try: out=c.enforce(plan.question,'rag',plan,result,index=index,complete=lambda m:pytest.fail('no time to audit'),prose_swaps=swaps) finally: mc.reset_deadline(token) assert out['delivery_check']['code']=='partial' and c.UNVERIFIED_NOTICE in out['answer'] assert plain in out['answer'] and readable not in out['answer'] def test_wording_fallback_never_rescues_a_different_request(): plan,result,index,swaps,readable,plain=_prose_fixture() out=c.enforce(plan.question,'rag',plan,result,index=index,complete=audit(plan_matches_request=False),prose_swaps=swaps) assert not out['accepted_for_delivery'] and out['reason']=='answer_rejected' def test_audit_evidence_is_grouped_without_losing_attribution(): from tools.answers.delivery_contract import audit_evidence rows={ 'a':{'id':'a','person':'Eva Nová','work_id':'w1','work_label':'Diplomová práca 2027','work_type':'diplomova', 'work_year':2027,'meeting_id':'m-long-internal-id','phase':'result','text':'Hotový model.', 'record_date':'2026-10-01','record_date_label':'2026-10-01'}, 'b':{'id':'b','person':'Eva Nová','work_id':'w1','work_label':'Diplomová práca 2027','work_type':'diplomova', 'work_year':2027,'phase':'planned','text':'Vyhodnoťte model.'}, 'c':{'id':'c','person':'Adam Nový','work_id':'w2','work_label':'Bakalárska práca 2025','work_type':'bakalarska', 'work_year':2025,'phase':'progress','text':'Zbiera dáta.','event_date':'2024-11-02'}, } groups=audit_evidence(rows,{'a':'E1','b':'E2','c':'E3'}) assert [(g['person'],g['work_label'],[u['id'] for u in g['evidence']]) for g in groups]==[ ('Eva Nová','Diplomová práca 2027',['E1','E2']),('Adam Nový','Bakalárska práca 2025',['E3'])] assert groups[0]['evidence'][0]=={'id':'E1','phase':'result','record_date':'2026-10-01','text':'Hotový model.'} assert groups[0]['evidence'][1]=={'id':'E2','phase':'planned','text':'Vyhodnoťte model.'} assert groups[1]['evidence'][0]['event_date']=='2024-11-02' assert 'meeting_id' not in str(groups) def test_verbatim_quoted_unit_is_sent_once_as_a_reference(): from tools.answers.delivery_contract import audit_evidence text='- Dotrénované 4 modely, slovakt5-base, slavict5-base, mt5-base, umt5-base.' rows={'a':{'id':'a','person':'Eva Nová','phase':'result','text':text}, 'b':{'id':'b','person':'Eva Nová','phase':'result','text':'- Model s augmentovanými dátami je o dosť lepší než pôvodný.'}} segments={'A1':'**Eva Nová**','A2':'- Bakalárska práca 2025; zaznamenaný výsledok: '+text[2:], 'A3':'Bakalárska práca 2025: Zlepšila model pomocou augmentovaných dát.'} units=audit_evidence(rows,{'a':'E1','b':'E2'},segments)[0]['evidence'] assert units[0]=={'id':'E1','phase':'result','text_quoted_verbatim_in':'A2'} # Paraphrased wording must still be compared with the source text. assert units[1]['text']==rows['b']['text'] def test_discovery_audit_receives_the_executed_decomposition(): from tools.answers import delivery_contract as dc plan,result,index=fixture() plan=plan.model_copy(update={'intent':'discovery','question':'Ktorí študenti robili s LLM a čo im fungovalo?'}) seen=[] def review(messages): seen.append(json.loads(messages[-1]['content'])) raise RuntimeError('stop') with pytest.raises(dc.ContractError): dc.semantic_check(plan.question,plan,result,complete=review) assert seen[0]['proposed_requirements']==[{'id':'R1','text':'Ktorí študenti robili s LLM'}, {'id':'R2','text':'čo im fungovalo'}] def test_stray_reference_in_a_long_list_is_dropped_not_used(): plan,result,index=fixture() model=audit(requirements=[{'description':'Results','question_segments':['Q1'], 'status':'answered','answer_segments':['A1'],'evidence_ids':['E1','A1']}]) assert c.enforce(plan.question,'rag',plan,result,index=index,complete=model)['accepted_for_delivery'] def test_answered_requirement_citing_only_unknown_units_has_no_support(): plan,result,index=fixture() model=audit(requirements=[{'description':'Results','question_segments':['Q1'], 'status':'answered','answer_segments':['A1'],'evidence_ids':['E999']}]) out=c.enforce(plan.question,'rag',plan,result,index=index,complete=model) assert out['delivery_check']['code']=='request_not_fulfilled' def test_doubt_disclosed_by_the_backend_notice_is_delivered_as_partial(): plan,result,index=fixture() note='Profil obsahuje aj záznamy bez úplného dátumu; nemožno spoľahlivo určiť, či sú novšie.' result={**result,'answer':result['answer']+'\n\n'+note,'presentation_notes':[note]} def doubt(segment): def model(messages): p=json.loads(messages[-1]['content']) ids=[s['id'] for s in p['answer_segments'] if segment(s['text'])] value={'plan_matches_request':True,'all_requirements_identified':True,'facts_supported':True,'clear':True, 'requirements':[{'description':'Latest meeting','question_segments':['Q1'],'status':'uncertain', 'answer_segments':ids,'evidence_ids':['E1']}]} return {'choices':[{'message':{'content':json.dumps(value)}}]} return model out=c.enforce(plan.question,'rag',plan,result,index=index,complete=doubt(lambda t:True)) assert out['accepted_for_delivery'] and out['delivery_check']['code']=='partial' assert c.UNCERTAIN_NOTICE in out['answer'] # The same doubt without the backend's own notice is still rejected. out=c.enforce(plan.question,'rag',plan,result,index=index,complete=doubt(lambda t:note not in t)) assert not out['accepted_for_delivery']