"""Cover the real comparison failures without relaxing factual evidence checks.""" import copy import json import sqlite3 import pytest from test_rag_work_content import payload, QUESTION, URL from test_quality_block import corpus, prepare, complete from tools.answer_evaluation.evidence_selection import resolve_selection, EvidenceSelectionError from tools.answer_evaluation.rag_evidence import rag_evidence_context from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues from tools.answer_evaluation.content_selection import fit_candidates from tools.answer_evaluation.question_intent import graph_needs_text from tools.answers import answer_delivery from tools.rag.rag_utils import build_context_text GOALS=['1. Vypracujte prehľad metód ohodnotenia dokumentov.', '2. Implementujte systém získavania informácií v slovenskom jazyku.', '3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy.', '4. Navrhnite možné zlepšenia presnosti vyhľadávania.'] @pytest.fixture def four_goals(payload): payload['sources']=payload['sources'][:1] payload['sources'][0]['text']='Sekcia: Diplomová práca 2021\nNázov: Vyhľadávanie informácií.\nZadanie:\n'+'\n'.join(GOALS) payload['context']=build_context_text(payload['sources']) return payload def id_answer(context): return {'status':'ok','answer':'','claims':[],'evidence_quotes':[], 'evidence_ids':[row['evidence_id'] for row in context['content_candidates']], 'source_urls':[]} @pytest.mark.parametrize('question',[QUESTION,'Aké je celé zadanie diplomovej práce Jána Holpa?']) def test_all_four_goals_are_delivered_with_exact_evidence_and_automatic_citation(four_goals,question): c=rag_evidence_context(question,four_goals);a=id_answer(c) assert len(a['evidence_ids'])==4 before=copy.deepcopy((a,c));expanded=resolve_selection(a,c) assert (a,c)==before and expanded['source_urls']==[URL] assert [q['quote'] for q in expanded['evidence_quotes']]==GOALS r=generate_verified_rag(question,four_goals,complete(a)) assert r['accepted_for_delivery'] and len(r['attempts'])==1 assert '3. Návrh a vypracovanie experimentov' in r['answer'] assert '4. Návrh možných zlepšení presnosti vyhľadávania.' in r['answer'] if r['answer_coverage']['scope']=='assignment': assert r['answer_coverage']['status']=='partial' assert 'assignment_completeness_unknown' in r['answer_coverage']['warnings'] else:assert r['answer_coverage']['status']=='supported' assert 'Výpis je neúplný' not in r['answer'] assert r['attempts'][0]['selected_answer']['evidence_ids']==a['evidence_ids'] def test_two_of_four_goals_cannot_be_silently_accepted_as_an_overview(four_goals): c=rag_evidence_context(QUESTION,four_goals);a=id_answer(c);a['evidence_ids']=a['evidence_ids'][:2] r=generate_verified_rag(QUESTION,four_goals,complete(a)) assert not r['accepted_for_delivery'] and len(r['attempts'])==2 assert all('missing_assignment_evidence' in attempt['issues'] for attempt in r['attempts']) def test_unknown_id_is_rejected_in_both_attempts(four_goals): a=id_answer(rag_evidence_context(QUESTION,four_goals));a['evidence_ids']=['Eunknown'] r=generate_verified_rag(QUESTION,four_goals,complete(a)) assert not r['accepted_for_delivery'] assert all(attempt['issues']==['unknown_evidence_id'] for attempt in r['attempts']) def test_ids_cannot_smuggle_additional_quotes_or_urls(four_goals): c=rag_evidence_context(QUESTION,four_goals);a=id_answer(c) both={**a,'evidence_quotes':[{'source_id':'S1','quote':GOALS[0]}]} with pytest.raises(EvidenceSelectionError,match='mixed_evidence_selection'): resolve_selection(both,c) a['source_urls']=['https://invented.test/result'] r=generate_verified_rag(QUESTION,four_goals,complete(a)) assert not r['accepted_for_delivery'] and 'unsupported_citation' in r['attempts'][-1]['issues'] def test_duplicate_ids_do_not_duplicate_facts(four_goals): c=rag_evidence_context(QUESTION,four_goals);a=id_answer(c);a['evidence_ids']*=2 resolved=resolve_selection(a,c) assert len(resolved['evidence_quotes'])==4 and resolved['source_urls']==[URL] a['evidence_ids'].reverse() assert [q['quote'] for q in resolve_selection(a,c)['evidence_quotes']]==GOALS def test_model_does_not_receive_duplicate_full_sources_or_need_to_copy_markdown(four_goals): def select(messages): c=json.loads(messages[1]['content'])['context'] assert 'content_passages' not in c and 'context' not in c assert all('text' not in row for row in c['sources']) assert all(row['required'] for row in c['content_candidates']) return complete(id_answer(c))(messages) assert generate_verified_rag(QUESTION,four_goals,select)['accepted_for_delivery'] def test_primary_goals_are_not_extended_with_the_task_backlog(four_goals): four_goals['sources'][0]['text']+='\nZásobník úloh:\n- Implementujte neskorší doplnkový modul.' four_goals['context']=build_context_text(four_goals['sources']) c=rag_evidence_context(QUESTION,four_goals) assert len(c['content_candidates'])==4 r=generate_verified_rag(QUESTION,four_goals,complete(id_answer(c))) assert r['accepted_for_delivery'] and 'doplnkový modul' not in r['answer'] @pytest.mark.parametrize('heading',['Napr.','Príklady:']) def test_illustrative_examples_after_assignment_are_not_extra_mandatory_goals(four_goals,heading): four_goals['sources'][0]['text']+='\n\n'+heading+'\n\n- osnovy pre výuku na základnej škole.' four_goals['context']=build_context_text(four_goals['sources']) c=rag_evidence_context(QUESTION,four_goals) assert len(c['content_candidates'])==4 r=generate_verified_rag(QUESTION,four_goals,complete(id_answer(c))) assert r['accepted_for_delivery'] and 'osnovy pre výuku' not in r['answer'] def test_qualified_goal_is_kept_whole(four_goals): text='- Implementujte parser. Výsledok sa smie použiť iba na testovacom korpuse; nejde o overenie produkcie.' four_goals['sources'][0]['text']+='\n'+text four_goals['context']=build_context_text(four_goals['sources']) c=rag_evidence_context(QUESTION,four_goals);r=generate_verified_rag(QUESTION,four_goals,complete(id_answer(c))) assert r['accepted_for_delivery'] assert 'iba na testovacom korpuse; nejde o overenie produkcie' in r['answer'] def test_budgeted_out_id_cannot_be_resolved_and_limited_overview_has_a_specific_notice(four_goals): c=rag_evidence_context(QUESTION,four_goals);original=c['content_candidates'][:] sent={'requirements':c['requirements'],'content_candidates':original} fit_candidates(sent,c,450,'cl100k_base') dropped=[row for row in original if row not in c['content_candidates']] assert dropped and c['truncated'] and 'answer_candidate_budget' in c['truncation_reasons'] with pytest.raises(EvidenceSelectionError,match='unknown_evidence_id'): resolve_selection({**id_answer(c),'evidence_ids':[dropped[0]['evidence_id']]},c) from tools.answer_evaluation.answer_coverage import answer_coverage,coverage_notices a=resolve_selection(id_answer(c),c) coverage=answer_coverage(a,c) assert coverage['status']=='partial' and 'overview_aspects_clipped' in coverage['warnings'] assert any('prehľad preto nemusí pokrývať všetky oblasti' in n for n in coverage_notices(coverage)) def test_an_indivisible_overlimit_goal_does_not_force_an_answer_the_model_never_received(four_goals): four_goals['sources'][0]['text']='Sekcia: Diplomová práca 2021\nZadanie:\n- Implementujte parser. '+('Platí iba pre testovací korpus. '*100) four_goals['context']=build_context_text(four_goals['sources']) c=rag_evidence_context(QUESTION,four_goals) assert not c['content_candidates'] and c['truncated'] a={'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]} r=generate_verified_rag(QUESTION,four_goals,complete(a)) assert r['accepted_for_delivery'] and r['status']=='no_answer' assert 'Časť podkladov bola pri príprave kontextu skrátená.' in r['answer'] @pytest.mark.parametrize('tail,want_year',[('',False),(' Uveď aj rok práce.',True)]) def test_year_filter_is_preserved_without_forcing_a_year_claim(corpus,tail,want_year): q='Aké problémy riešila Eva Nová pri diplomovej práci v roku 2024?'+tail p=prepare(corpus,q);c=rag_evidence_context(q,p) assert c['requirements']['years']==[2024] and c['requirements']['year']==want_year assert all(row['year']==2024 for row in c['content_candidates']) if not want_year: r=generate_verified_rag(q,p,complete(id_answer(c))) assert r['accepted_for_delivery'] and r['claims']==[] def test_comparison_cannot_replace_one_persons_evidence_with_another(corpus): q='Porovnaj výsledky Evy Novej a Borisa Nového.' p=prepare(corpus,q);c=rag_evidence_context(q,p);a=id_answer(c) a['evidence_ids']=[row['evidence_id'] for row in c['content_candidates'] if row['person']=='Eva Nová'] r=generate_verified_rag(q,p,complete(a)) assert not r['accepted_for_delivery'] and 'missing_progress_evidence' in r['attempts'][-1]['issues'] def test_partial_refusal_gets_corrected_without_inventing_the_missing_persons_evidence(corpus): db,_=corpus with sqlite3.connect(db) as conn: conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.' WHERE document_path LIKE '%boris_novy%'") q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?' p=prepare(corpus,q);c=rag_evidence_context(q,p);calls=[] def select(messages): calls.append(copy.deepcopy(messages)) assert 'Doložené časti po osobách' in messages[0]['content'] assert 'Eva Nová' not in messages[0]['content'] and 'Boris Nový' not in messages[0]['content'] return complete({'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]} if len(calls)==1 else id_answer(c))(messages) r=generate_verified_rag(q,p,select) assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial' and len(calls)==2 assert 'rejected_supported_content_comparison' in r['attempts'][0]['issues'] assert 'Boris Nový' in r['answer'] and 'chýbajú údaje o riešených problémoch' in r['answer'] assert len(r['source_urls'])==2 and r['claims']==[] def test_graph_narrative_request_gets_specific_rag_guidance_without_a_model(monkeypatch): def forbidden(*args,**kwargs):pytest.fail('Graph/text mismatch must not invoke retrieval or a model') monkeypatch.setattr(answer_delivery,'search_graph',forbidden) q='Aké problémy riešili Ján Pták a Ján Malinovský pri diplomovom projekte v roku 2026 a aké experimenty mali pripraviť?' r=answer_delivery.deliver_answer(q,engine='graphrag',complete=forbidden) assert r['status']=='ambiguous' and r['reason']=='text_evidence_required' assert r['accepted_for_delivery'] and '/rag' in r['answer'] and r['claims']==[] assert not graph_needs_text('Ktorí študenti majú tag RAG a zároveň diplomový projekt z roku 2026?') assert not graph_needs_text('Aké spoločné kľúčové slová majú Ján Pták a Ján Malinovský?')