208 lines
13 KiB
Python
208 lines
13 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from tools.answer_evaluation.evaluate_verified_rag import binding_matches, evaluate_case
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
|
from tools.answer_evaluation.verified_rag import rag_answer_issues, generate_verified_rag
|
|
from tools.rag.rag_utils import build_context_text
|
|
|
|
|
|
def payload(text, *, categories=None):
|
|
sources = [{'source_id':'S1', 'title':'Alena Nová',
|
|
'document_path':'pages/students/2020/alena_nova/README.md',
|
|
'source_url':'https://example.test/alena', 'categories':categories or [],
|
|
'text':text}]
|
|
return {'status':'ok', 'complete':None, 'sources':sources, 'context':build_context_text(sources)}
|
|
|
|
|
|
def refusal():
|
|
return {'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]}
|
|
|
|
|
|
@pytest.mark.parametrize('question', ['V ktorom roku začal študovať Alena Nová?',
|
|
'Aký je rok začiatku štúdia osoby Alena Nová?'])
|
|
def test_explicit_study_start_cannot_be_falsely_refused(question):
|
|
p = payload('Dokument: Alena Nová\n\n*Rok začiatku štúdia:* 2020')
|
|
context = rag_evidence_context(question, p)
|
|
assert context['requirements']['start_year']
|
|
assert 'rejected_supported_academic_answer' in rag_answer_issues(refusal(), context)
|
|
result = generate_verified_rag(question, p, lambda messages: {'choices':[{'message':{'content':json.dumps(refusal())}}]})
|
|
assert not result['accepted_for_delivery']
|
|
|
|
|
|
def test_study_start_never_comes_from_cohort_or_academic_year():
|
|
c = rag_evidence_context('Kedy začal študovať Alena Nová?',
|
|
payload('Sekcia: Bakalárska práca 2023\nNázov: Overené modelovanie textu', categories=['bp2023']))
|
|
assert not any('start_year' in row for row in c['facts'])
|
|
assert rag_answer_issues(refusal(), c) == []
|
|
|
|
|
|
@pytest.mark.parametrize('year,unsupported', [('3000',False),('12345',True)])
|
|
def test_explicit_year_is_honored_or_rejected_even_in_pre_rok_form(year,unsupported):
|
|
c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová pre rok '+year+'?',
|
|
payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu'))
|
|
assert c['requirements']['unsupported_year'] == unsupported
|
|
answer = {'status':'ok','claims':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025,
|
|
'title':'Overené modelovanie textu'}], 'source_urls':['https://example.test/alena']}
|
|
assert ('unsupported_requested_year' if unsupported else 'wrong_requested_academic_year') in rag_answer_issues(answer,c)
|
|
|
|
|
|
def test_multiline_title_and_unlabelled_title_preceding_assignment():
|
|
p = payload('Sekcia: Diplomová práca 2025\nNázov práce:\n\nOverené modelovanie textu\n\nZadanie:\n1. Skúmajte dáta.')
|
|
c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová?', p)
|
|
assert any(row.get('title')=='Overené modelovanie textu' and row['year']==2025 for row in c['facts'])
|
|
p = payload('Sekcia: Diplomová práca 2025\nOverené modelovanie textu\nCieľ:\nZlepšiť model.')
|
|
assert any(row.get('title')=='Overené modelovanie textu' for row in rag_evidence_context('Práca?',p)['facts'])
|
|
|
|
|
|
@pytest.mark.parametrize('text', [
|
|
'Sekcia: Diplomová práca 2025\nStretnutie po úspešných experimentoch\nStav:\nHotovo.',
|
|
'Sekcia: Diplomová práca 2025\nTéma:\nOverené modelovanie tex\n[OBSAH ZDROJA OREZANÝ]',
|
|
'Sekcia: Diplomová práca 2025\nNázov:\n\nĎALŠÍ DÔKAZ Z ROVNAKEJ ŠTUDENTSKEJ VETVY 2\nSekcia: Diplomová práca 2025\nStretnutie 12.3.'
|
|
])
|
|
def test_notes_clipped_titles_and_frame_labels_never_become_titles(text):
|
|
assert not any(row.get('title') for row in rag_evidence_context('Práca?',payload(text))['facts'])
|
|
|
|
|
|
def test_sibling_scope_and_preparation_cannot_inherit_previous_work():
|
|
p = payload('# Diplomová práca 2025\nNázov: Overené modelovanie textu\n'
|
|
'# Poznámky\nNázov: Úplne iný dokument\n'
|
|
'# Príprava na diplomový projekt 2026\nNázov: Ďalší prípravný dokument')
|
|
c = rag_evidence_context('Aký je názov diplomovej práce osoby Alena Nová?',p)
|
|
titles = {row.get('title') for row in c['facts'] if row.get('work_type')}
|
|
assert titles == {'Overené modelovanie textu'}
|
|
|
|
|
|
def test_typed_root_proposal_needs_one_matching_academic_scope():
|
|
p = payload('Dokument: Alena Nová\nNávrh na názov DP:\nOverené modelovanie textu\n'
|
|
'Sekcia: Diplomová práca 2025\nStretnutie 12.3.')
|
|
c = rag_evidence_context('Práca?',p)
|
|
assert any(r.get('title')=='Overené modelovanie textu' and r['year']==2025 for r in c['facts'])
|
|
p['sources'][0]['text'] += '\nSekcia: Diplomová práca 2024\nStretnutie 5.3.'
|
|
p['context'] = build_context_text(p['sources'])
|
|
assert not any(r.get('title') for r in rag_evidence_context('Práca?',p)['facts'])
|
|
|
|
|
|
def test_quote_from_another_student_or_only_metadata_is_not_an_answer():
|
|
p = payload('Rok začiatku štúdia: 2020\nPoužíva Python.')
|
|
other = {**p['sources'][0], 'source_id':'S2','title':'Boris Nový',
|
|
'document_path':'pages/students/2020/boris_novy/README.md',
|
|
'source_url':'https://example.test/boris', 'text':'Používa Rust.'}
|
|
p['sources'].append(other); p['context']=build_context_text(p['sources'])
|
|
c = rag_evidence_context('Aký jazyk používa Alena Nová?',p)
|
|
a = {'status':'ok','claims':[],'source_urls':['https://example.test/boris'],
|
|
'evidence_quotes':[{'source_id':'S2','quote':'Používa Rust.'}]}
|
|
assert 'unrequested_quote_person' in rag_answer_issues(a,c)
|
|
a.update(source_urls=['https://example.test/alena'], evidence_quotes=[{'source_id':'S1','quote':'Názov dokumentu: Alena Nová'}])
|
|
assert 'metadata_quote_does_not_answer_fact' in rag_answer_issues(a,c)
|
|
|
|
|
|
def test_evaluator_rejects_cross_person_join_wrong_tuple_and_false_refusal():
|
|
expected = {'person':'Alena Nová', 'title':'Overené modelovanie textu','year':2025,'work_type':'diploma_thesis'}
|
|
assert not binding_matches(expected,[{'person':'Alena Nová','title':expected['title']},
|
|
{'person':'Boris Nový','year':2025,'work_type':'diploma_thesis'}])
|
|
p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu')
|
|
case = {'id':'fixture','category':'fact_title','question':'Aký je názov diplomovej práce osoby Alena Nová?',
|
|
'should_answer':True,'expected_bindings':[expected], 'expected_source_urls':['https://example.test/alena'],
|
|
'expected_answer_contains':['Overené modelovanie textu']}
|
|
refused = {'status':'no_answer','accepted_for_delivery':True,'answer':'Nenájdené.','claims':[],'source_urls':[],'attempts':[]}
|
|
assert 'false_no_answer' in evaluate_case(case,p,refused)['issues']
|
|
bad = {**refused,'status':'ok','answer':'Overené modelovanie textu','source_urls':case['expected_source_urls'],
|
|
'claims':[{**expected,'year':2024}]}
|
|
assert 'unsupported_person_work_type_year_binding' in evaluate_case(case,p,bad)['issues']
|
|
|
|
|
|
def test_evaluator_accepts_extra_supported_project_but_rejects_wrong_rendered_year():
|
|
from tools.answer_evaluation.verified_rag import render_rag_answer
|
|
p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\n'
|
|
'Sekcia: Diplomový projekt 2024\nStretnutie 12.3.')
|
|
question = 'Aký je rok práce osoby Alena Nová?'
|
|
context = rag_evidence_context(question,p)
|
|
claims = [{'person':'Alena Nová','work_type':'diploma_thesis','year':2025},
|
|
{'person':'Alena Nová','work_type':'diploma_project','year':2024}]
|
|
parsed = {'status':'ok','claims':claims,'source_urls':['https://example.test/alena'],'evidence_quotes':[]}
|
|
delivered = {**parsed,'accepted_for_delivery':True,'answer':render_rag_answer(parsed,context),
|
|
'attempts':[{'parsed_answer':parsed,'issues':[]}]}
|
|
case = {'id':'fixture','category':'fact_work_year','question':question,'should_answer':True,
|
|
'expected_source_urls':parsed['source_urls'], 'expected_answer_contains':['2025'],
|
|
'expected_bindings':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}]}
|
|
assert evaluate_case(case,p,delivered)['passed']
|
|
delivered['answer'] += '\nPráca bola ukončená v roku 2099.'
|
|
assert 'delivered_text_differs_from_verified_renderer' in evaluate_case(case,p,delivered)['issues']
|
|
|
|
|
|
@pytest.mark.parametrize('reference', ['https://library.example/docs', '/topics/library',
|
|
'../manual', '#installation', '[guide]: /guide'])
|
|
def test_literal_source_references_are_allowed_without_replacing_wiki_citation(reference):
|
|
from tools.answer_evaluation.verified_rag import quoted_reference_urls, render_rag_answer
|
|
from tools.answer_evaluation.context_policy import citation_check, allowed_urls
|
|
quote = ('Dokumentácia: '+reference if not reference.startswith('[guide]') else reference)
|
|
if reference.startswith(('/', '.', '#')):
|
|
quote = '[Dokumentácia]('+reference+')'
|
|
c = rag_evidence_context('Aká dokumentácia je uvedená?',payload(quote))
|
|
a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
|
|
'evidence_quotes':[{'source_id':'S1','quote':quote}]}
|
|
assert rag_answer_issues(a,c) == []
|
|
rendered = {**a,'answer':render_rag_answer(a,c)}
|
|
assert citation_check(rendered,allowed_urls(c)|quoted_reference_urls(a,c))['citations_ok']
|
|
refs = quoted_reference_urls(a,c)
|
|
assert refs
|
|
a['source_urls'].extend(refs)
|
|
assert 'unsupported_citation' in rag_answer_issues(a,c)
|
|
|
|
|
|
@pytest.mark.parametrize('quote', ['Pozri https://unknown.example/doc',
|
|
'[manuál](/unknown)', '<a href="/unknown">manuál</a>'])
|
|
def test_quote_cannot_borrow_an_unknown_or_other_source_reference(quote):
|
|
p = payload('Presný obsah bez odkazu.')
|
|
p['sources'].append({**p['sources'][0],'source_id':'S2','title':'Iný dokument',
|
|
'source_url':'https://example.test/other','text':quote})
|
|
p['context'] = build_context_text(p['sources'])
|
|
c = rag_evidence_context('Aký je obsah dokumentu?',p)
|
|
a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
|
|
'evidence_quotes':[{'source_id':'S1','quote':quote}]}
|
|
assert 'unsupported_quote_citation' in rag_answer_issues(a,c)
|
|
|
|
|
|
def test_active_url_schemes_and_noncontiguous_quotes_are_rejected():
|
|
c = rag_evidence_context('Aký je obsah dokumentu?',payload('[odkaz](javascript:alert(1))\nPrvý fakt.\nMedziľahlý text.\nDruhý fakt.'))
|
|
a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
|
|
'evidence_quotes':[{'source_id':'S1','quote':'[odkaz](javascript:alert(1))'}]}
|
|
assert 'unsupported_quote_citation' in rag_answer_issues(a,c)
|
|
a['evidence_quotes']=[{'source_id':'S1','quote':'Prvý fakt.\nDruhý fakt.'}]
|
|
assert 'unsupported_evidence_quote' in rag_answer_issues(a,c)
|
|
|
|
|
|
def test_general_question_repairs_unrequested_claims_and_sends_only_quote_evidence():
|
|
p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\nPoužíva Python.')
|
|
bad = {'status':'ok','answer':'','claims':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}],
|
|
'evidence_quotes':[{'source_id':'S1','quote':'Používa Python.'}],
|
|
'source_urls':['https://example.test/alena']}
|
|
good = {**bad,'claims':[]}
|
|
outputs = [bad, good]; calls=[]
|
|
def complete(messages):
|
|
calls.append(json.loads(messages[1]['content']))
|
|
return {'choices':[{'message':{'content':json.dumps(outputs[len(calls)-1])}}]}
|
|
delivered = generate_verified_rag('Aký jazyk používa Alena Nová?',p,complete)
|
|
assert delivered['accepted_for_delivery'] and len(calls)==2
|
|
assert 'unrequested_structured_claims' in delivered['attempts'][0]['issues']
|
|
assert all('facts' not in call['context'] for call in calls)
|
|
assert delivered['claims']==[] and 'Používa Python.' in delivered['answer']
|
|
|
|
|
|
def test_evaluator_accepts_exact_quoted_reference_but_rejects_invented_rendered_link():
|
|
from tools.answer_evaluation.verified_rag import render_rag_answer
|
|
p = payload('Knižnica Library: https://library.example/docs')
|
|
question = 'Aká knižnica je uvedená?'
|
|
c = rag_evidence_context(question,p)
|
|
parsed = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
|
|
'evidence_quotes':[{'source_id':'S1','quote':'Knižnica Library: https://library.example/docs'}]}
|
|
delivered = {**parsed,'accepted_for_delivery':True,'answer':render_rag_answer(parsed,c),
|
|
'attempts':[{'parsed_answer':parsed,'issues':[]}]}
|
|
case = {'id':'reference','category':'specific_fact','question':question,'should_answer':True,
|
|
'expected_bindings':[],'expected_source_urls':parsed['source_urls'],'expected_answer_contains':['Library']}
|
|
assert evaluate_case(case,p,delivered)['passed']
|
|
delivered['answer']+=' https://invented.example/'
|
|
assert 'delivered_unsupported_citation' in evaluate_case(case,p,delivered)['issues']
|