import json import pytest from tools.answer_evaluation.evaluate_verified_rag import binding_matches, evaluate_case from tools.answer_evaluation.rag_evidence import rag_evidence_context from tools.answer_evaluation.verified_rag import rag_answer_issues, generate_verified_rag from tools.rag.rag_utils import build_context_text def payload(text, *, categories=None): sources = [{'source_id':'S1', 'title':'Alena Nová', 'document_path':'pages/students/2020/alena_nova/README.md', 'source_url':'https://example.test/alena', 'categories':categories or [], 'text':text}] return {'status':'ok', 'complete':None, 'sources':sources, 'context':build_context_text(sources)} def refusal(): return {'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]} @pytest.mark.parametrize('question', ['V ktorom roku začal študovať Alena Nová?', 'Aký je rok začiatku štúdia osoby Alena Nová?']) def test_explicit_study_start_cannot_be_falsely_refused(question): p = payload('Dokument: Alena Nová\n\n*Rok začiatku štúdia:* 2020') context = rag_evidence_context(question, p) assert context['requirements']['start_year'] assert 'rejected_supported_academic_answer' in rag_answer_issues(refusal(), context) result = generate_verified_rag(question, p, lambda messages: {'choices':[{'message':{'content':json.dumps(refusal())}}]}) assert not result['accepted_for_delivery'] def test_study_start_never_comes_from_cohort_or_academic_year(): c = rag_evidence_context('Kedy začal študovať Alena Nová?', payload('Sekcia: Bakalárska práca 2023\nNázov: Overené modelovanie textu', categories=['bp2023'])) assert not any('start_year' in row for row in c['facts']) assert rag_answer_issues(refusal(), c) == [] @pytest.mark.parametrize('year,unsupported', [('3000',False),('12345',True)]) def test_explicit_year_is_honored_or_rejected_even_in_pre_rok_form(year,unsupported): c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová pre rok '+year+'?', payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu')) assert c['requirements']['unsupported_year'] == unsupported answer = {'status':'ok','claims':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025, 'title':'Overené modelovanie textu'}], 'source_urls':['https://example.test/alena']} assert ('unsupported_requested_year' if unsupported else 'wrong_requested_academic_year') in rag_answer_issues(answer,c) def test_multiline_title_and_unlabelled_title_preceding_assignment(): p = payload('Sekcia: Diplomová práca 2025\nNázov práce:\n\nOverené modelovanie textu\n\nZadanie:\n1. Skúmajte dáta.') c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová?', p) assert any(row.get('title')=='Overené modelovanie textu' and row['year']==2025 for row in c['facts']) p = payload('Sekcia: Diplomová práca 2025\nOverené modelovanie textu\nCieľ:\nZlepšiť model.') assert any(row.get('title')=='Overené modelovanie textu' for row in rag_evidence_context('Práca?',p)['facts']) @pytest.mark.parametrize('text', [ 'Sekcia: Diplomová práca 2025\nStretnutie po úspešných experimentoch\nStav:\nHotovo.', 'Sekcia: Diplomová práca 2025\nTéma:\nOverené modelovanie tex\n[OBSAH ZDROJA OREZANÝ]', 'Sekcia: Diplomová práca 2025\nNázov:\n\nĎALŠÍ DÔKAZ Z ROVNAKEJ ŠTUDENTSKEJ VETVY 2\nSekcia: Diplomová práca 2025\nStretnutie 12.3.' ]) def test_notes_clipped_titles_and_frame_labels_never_become_titles(text): assert not any(row.get('title') for row in rag_evidence_context('Práca?',payload(text))['facts']) def test_sibling_scope_and_preparation_cannot_inherit_previous_work(): p = payload('# Diplomová práca 2025\nNázov: Overené modelovanie textu\n' '# Poznámky\nNázov: Úplne iný dokument\n' '# Príprava na diplomový projekt 2026\nNázov: Ďalší prípravný dokument') c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová?',p) titles = {row.get('title') for row in c['facts'] if row.get('work_type')} assert titles == {'Overené modelovanie textu'} def test_typed_root_proposal_needs_one_matching_academic_scope(): p = payload('Dokument: Alena Nová\nNávrh na názov DP:\nOverené modelovanie textu\n' 'Sekcia: Diplomová práca 2025\nStretnutie 12.3.') c = rag_evidence_context('Práca?',p) assert any(r.get('title')=='Overené modelovanie textu' and r['year']==2025 for r in c['facts']) p['sources'][0]['text'] += '\nSekcia: Diplomová práca 2024\nStretnutie 5.3.' p['context'] = build_context_text(p['sources']) assert not any(r.get('title') for r in rag_evidence_context('Práca?',p)['facts']) def test_quote_from_another_student_or_only_metadata_is_not_an_answer(): p = payload('Rok začiatku štúdia: 2020\nPoužíva Python.') other = {**p['sources'][0], 'source_id':'S2','title':'Boris Nový', 'document_path':'pages/students/2020/boris_novy/README.md', 'source_url':'https://example.test/boris', 'text':'Používa Rust.'} p['sources'].append(other); p['context']=build_context_text(p['sources']) c = rag_evidence_context('Aký jazyk používa Alena Nová?',p) a = {'status':'ok','claims':[],'source_urls':['https://example.test/boris'], 'evidence_quotes':[{'source_id':'S2','quote':'Používa Rust.'}]} assert 'unrequested_quote_person' in rag_answer_issues(a,c) a.update(source_urls=['https://example.test/alena'], evidence_quotes=[{'source_id':'S1','quote':'Názov dokumentu: Alena Nová'}]) assert 'metadata_quote_does_not_answer_fact' in rag_answer_issues(a,c) def test_evaluator_rejects_cross_person_join_wrong_tuple_and_false_refusal(): expected = {'person':'Alena Nová', 'title':'Overené modelovanie textu','year':2025,'work_type':'diploma_thesis'} assert not binding_matches(expected,[{'person':'Alena Nová','title':expected['title']}, {'person':'Boris Nový','year':2025,'work_type':'diploma_thesis'}]) p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu') case = {'id':'fixture','category':'fact_title','question':'Aký je názov diplomovej práce osoby Alena Nová?', 'should_answer':True,'expected_bindings':[expected], 'expected_source_urls':['https://example.test/alena'], 'expected_answer_contains':['Overené modelovanie textu']} refused = {'status':'no_answer','accepted_for_delivery':True,'answer':'Nenájdené.','claims':[],'source_urls':[],'attempts':[]} assert 'false_no_answer' in evaluate_case(case,p,refused)['issues'] bad = {**refused,'status':'ok','answer':'Overené modelovanie textu','source_urls':case['expected_source_urls'], 'claims':[{**expected,'year':2024}]} assert 'unsupported_person_work_type_year_binding' in evaluate_case(case,p,bad)['issues'] def test_evaluator_accepts_extra_supported_project_but_rejects_wrong_rendered_year(): from tools.answer_evaluation.verified_rag import render_rag_answer p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\n' 'Sekcia: Diplomový projekt 2024\nStretnutie 12.3.') question = 'Aký je rok práce osoby Alena Nová?' context = rag_evidence_context(question,p) claims = [{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}, {'person':'Alena Nová','work_type':'diploma_project','year':2024}] parsed = {'status':'ok','claims':claims,'source_urls':['https://example.test/alena'],'evidence_quotes':[]} delivered = {**parsed,'accepted_for_delivery':True,'answer':render_rag_answer(parsed,context), 'attempts':[{'parsed_answer':parsed,'issues':[]}]} case = {'id':'fixture','category':'fact_work_year','question':question,'should_answer':True, 'expected_source_urls':parsed['source_urls'], 'expected_answer_contains':['2025'], 'expected_bindings':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}]} assert evaluate_case(case,p,delivered)['passed'] delivered['answer'] += '\nPráca bola ukončená v roku 2099.' assert 'delivered_text_differs_from_verified_renderer' in evaluate_case(case,p,delivered)['issues']