159 lines
9.6 KiB
Python
159 lines
9.6 KiB
Python
"""Concise presentation must keep factual bindings, qualifications and gaps."""
|
|
import copy
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from test_quality_block import corpus, prepare, complete
|
|
from test_evidence_selection import update_eva, selection
|
|
from tools.answer_evaluation.context_policy import answer_context_issues
|
|
from tools.answer_evaluation.content_selection import facet_lines
|
|
from tools.answer_evaluation.graph_context import graph_context
|
|
from tools.answer_evaluation.presentation import comparison_table
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
|
from tools.answer_evaluation.render_answer import render_graph_answer
|
|
from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues
|
|
|
|
|
|
def test_missing_aspects_share_one_specific_notice_without_empty_sections(corpus):
|
|
update_eva(corpus,'Stav:\n- Implementovaný parser.')
|
|
q='Aké výsledky, problémy a plánované experimenty má diplomová práca Evy Novej?'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
|
result=generate_verified_rag(q,p,complete(selection(c)))
|
|
assert result['accepted_for_delivery'] and result['answer_coverage']['status']=='partial'
|
|
text=result['answer']
|
|
assert 'Implementovaný parser.' in text
|
|
assert text.count('V poskytnutých podkladoch chýbajú údaje')==1
|
|
assert 'riešených problémoch, plánovaných úlohách a plánovaných experimentoch' in text
|
|
assert '**Riešené problémy:**' not in text and '**Plánované experimenty' not in text
|
|
assert 'Táto časť odpovede' not in text and '\n\n\n' not in text
|
|
|
|
|
|
def test_short_comparison_aligns_people_and_different_results(corpus):
|
|
q='Porovnaj výsledky Evy Novej a Borisa Nového.'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p); a=selection(c)
|
|
before=copy.deepcopy((a,c))
|
|
lines=facet_lines(a,c); text='\n'.join(lines)
|
|
assert text.startswith('| Oblasť | Eva Nová — diplomová práca (2024) | Boris Nový — bakalárska práca (2023) |')
|
|
row=next(line for line in lines if line.startswith('| Zaznamenané výsledky |'))
|
|
assert 'BERT' in row.split('|')[2] and 'Python' in row.split('|')[3]
|
|
assert 'Konzultácia' not in text and 'Pripravte experiment' not in text
|
|
assert 'lepš' not in text and (a,c)==before
|
|
r=generate_verified_rag(q,p,complete(a))
|
|
assert r['accepted_for_delivery'] and len(r['source_urls'])==2
|
|
|
|
|
|
@pytest.mark.parametrize('literal',[
|
|
'Implementovaný parser. Výsledok platí iba pre malý korpus. '+('Technický opis. '*70),
|
|
'Implementovaný parser pre `a | b` iba na testovacích dátach.',
|
|
'Implementovaný parser podľa https://example.test/method iba na testovacích dátach.',
|
|
])
|
|
def test_complex_comparisons_fall_back_without_losing_qualifiers_or_literals(corpus,literal):
|
|
update_eva(corpus,'Výsledky:\n- '+literal)
|
|
q='Porovnaj výsledky Evy Novej a Borisa Nového.'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p); a=selection(c)
|
|
text='\n'.join(facet_lines(a,c))
|
|
assert not text.startswith('| Oblasť |') and literal.strip() in text
|
|
assert 'Boris Nový' in text and 'Python' in text
|
|
assert all('Ďalšie doložené' not in line for line in text.splitlines())
|
|
|
|
|
|
def test_shared_excerpt_is_not_repeated_under_each_matching_facet(corpus):
|
|
update_eva(corpus,'Výsledky:\n- Vyhodnotená presnosť modelu v experimente.')
|
|
q='Aké experimenty a výsledky má diplomová práca Evy Novej?'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
|
result=generate_verified_rag(q,p,complete(selection(c)))
|
|
assert result['accepted_for_delivery']
|
|
assert result['answer'].count('Vyhodnotená presnosť modelu v experimente.')==1
|
|
assert 'chýbajú údaje' not in result['answer'] and 'Doložené vyššie' not in result['answer']
|
|
assert 'Zaznamenaný výsledok:' in result['answer'] or '**Zaznamenané výsledky:**' in result['answer']
|
|
|
|
|
|
def test_multiline_table_is_bounded_and_does_not_rewrite_source_html():
|
|
table=comparison_table(['Oblasť','Eva','Boris'],[('Výsledky',['Prvý výsledok.\nDruhý výsledok.','Parser.'])])
|
|
assert table and 'Prvý výsledok.<br>Druhý výsledok.' in table[-1]
|
|
assert comparison_table(['Oblasť','Eva'],[('Výsledky',['<b>Prvý výsledok.</b>'])]) is None
|
|
|
|
|
|
def test_shared_result_precedes_people_and_citations_without_repeating_tags():
|
|
people=['Ján Pták','Ján Malinovský']; urls=['https://example.test/ptak','https://example.test/malinovsky']
|
|
claims=[{'person':person,'keyword':tag} for person in people for tag in ['NLP','RAG']]
|
|
c={'status':'ok','complete':True,'truncated':False,'facts':[
|
|
{**claim,'source_url':urls[people.index(claim['person'])]} for claim in claims],
|
|
'sources':[{'source_url':u} for u in urls], 'requested_people':people,
|
|
'shared_properties':[{'field':'keyword','value':tag,'people':people} for tag in ['NLP','RAG']]}
|
|
a={'status':'ok','claims':claims,'source_urls':urls}
|
|
r=render_graph_answer(a,c); text=r['answer']
|
|
assert text.startswith('Spoločné kľúčové slová: **NLP**, **RAG**.')
|
|
assert text.index('RAG')<text.index('Doložené pri:')<text.index('Zdroje:')
|
|
assert all(text.count('**'+tag+'**')==1 for tag in ['NLP','RAG'])
|
|
assert all(person in text for person in people)
|
|
assert not answer_context_issues(r,c)
|
|
|
|
|
|
def test_negative_names_failed_year_but_does_not_license_positive_year_claims():
|
|
c={'status':'no_answer','facts':[],'sources':[],'requested_people':['Martin Šarišský'],
|
|
'requested_years':[1999],'complete':True}
|
|
a=render_graph_answer({'status':'no_answer','claims':[],'source_urls':[]},c)
|
|
assert 'Martin Šarišský' in a['answer'] and 'z roku **1999**' in a['answer']
|
|
assert not answer_context_issues(a,c)
|
|
for year in [1999,2099]:
|
|
bad={**a,'answer':a['answer']+f' Práca bola dokončená v roku {year}.'}
|
|
assert 'unsupported_prose_number' in answer_context_issues(bad,c)
|
|
|
|
|
|
def test_start_year_is_not_rendered_as_a_missing_academic_work_year():
|
|
c=graph_context({'status':'no_answer','reason':'not_found','facts':[],'sources':[],'aggregation':None,
|
|
'complete':True,'constraints':{'years':[1999],'mentions_start_year':True},
|
|
'resolved_entities':[{'kind':'Person','name':'Martin Šarišský'}]})
|
|
a=render_graph_answer({'status':'no_answer','claims':[],'source_urls':[]},c)
|
|
assert 'záznam práce' not in a['answer'] and '1999' not in a['answer']
|
|
|
|
|
|
def test_partial_overview_retains_supported_person_and_names_missing_description(corpus):
|
|
db,raw=corpus
|
|
with sqlite3.connect(db) as conn:
|
|
conn.execute("UPDATE chunks SET text='Názov: Ukážkový názov.' WHERE document_path LIKE '%boris_novy%'")
|
|
q='Porovnaj obsah diplomovej práce Evy Novej a bakalárskej práce Borisa Nového.'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
|
assert not c['requirements'].get('facets')
|
|
passage=next(p for p in c['content_passages'] if p['person']=='Eva Nová')
|
|
a={'status':'ok','answer':'','claims':[], 'source_urls':[s['source_url'] for s in c['sources']],
|
|
'evidence_quotes':[{'source_id':passage['source_id'],'passage_id':passage['passage_id'],
|
|
'quote':'Implementujte systém vyhľadávania.'}]}
|
|
assert not rag_answer_issues(a,c)
|
|
result=generate_verified_rag(q,p,complete(a))
|
|
assert result['accepted_for_delivery'] and result['answer_coverage']['status']=='partial'
|
|
assert 'Implementácia systému vyhľadávania' in result['answer']
|
|
assert 'Boris Nový: v poskytnutých podkladoch chýba opis požadovanej práce alebo projektu.' in result['answer']
|
|
assert 'Výpis je neúplný' not in result['answer']
|
|
|
|
|
|
def test_available_description_cannot_be_silently_omitted(corpus):
|
|
q='Porovnaj obsah diplomovej práce Evy Novej a bakalárskej práce Borisa Nového.'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
|
passage=next(p for p in c['content_passages'] if p['person']=='Eva Nová')
|
|
a={'status':'ok','answer':'','claims':[], 'source_urls':[s['source_url'] for s in c['sources']],
|
|
'evidence_quotes':[{'source_id':passage['source_id'],'passage_id':passage['passage_id'],
|
|
'quote':'Implementujte systém vyhľadávania.'}]}
|
|
assert 'missing_bound_content_evidence' in rag_answer_issues(a,c)
|
|
|
|
|
|
@pytest.mark.parametrize('aspect',['obsah','zameranie'])
|
|
def test_overview_comparison_keeps_each_work_in_its_own_column(corpus,aspect):
|
|
q=f'Porovnaj {aspect} diplomovej práce Evy Novej a bakalárskej práce Borisa Nového.'
|
|
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
|
a={'status':'ok','answer':'','claims':[], 'source_urls':[s['source_url'] for s in c['sources']],
|
|
'evidence_quotes':[{'source_id':passage['source_id'],'passage_id':passage['passage_id'],'quote':text}
|
|
for person,text in [('Eva Nová','Implementujte systém vyhľadávania.'),
|
|
('Boris Nový','Implementujte parser.')]
|
|
for passage in c['content_passages'] if passage['person']==person and text in passage['text']]}
|
|
result=generate_verified_rag(q,p,complete(a))
|
|
assert result['accepted_for_delivery'] and result['answer_coverage']['status']=='supported'
|
|
assert result['answer'].startswith('| Oblasť | Diplomová práca Evy Novej (2024) | Bakalárska práca Borisa Nového (2023) |')
|
|
row=next(line for line in result['answer'].splitlines() if line.startswith('| Zameranie podľa podkladov |'))
|
|
assert 'vyhľadávania' in row.split('|')[2] and 'parser' in row.split('|')[3]
|
|
assert 'Pripravte experiment' not in row and len(result['source_urls'])==2
|
|
refused={**a,'status':'no_answer','evidence_quotes':[],'source_urls':[]}
|
|
assert 'rejected_supported_work_description' in rag_answer_issues(refused,c)
|