268 lines
14 KiB
Python
268 lines
14 KiB
Python
"""Name correction must preserve constraints and fail closed before retrieval."""
|
|
import copy
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.retrieval.person_names import resolve_person_names, sqlite_people
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
|
|
|
|
def person(name, slug):
|
|
return {'kind':'Person','name':name,'element_id':slug,
|
|
'source_document':f'pages/students/2022/{slug}/README.md'}
|
|
|
|
|
|
@pytest.fixture
|
|
def people():
|
|
return [person('Ján Holp','jan_holp'),person('Ondrej Megela','ondrej_megela'),
|
|
person('Ján Malinovský','jan_malinovsky'),person('Ján Pták','jan_ptak'),
|
|
person('Martin Šarišský','martin_sarissky'),person('Martin Jančura','martin_jancura')]
|
|
|
|
|
|
@pytest.mark.parametrize('mention,name',[
|
|
('Jan Holp','Ján Holp'),('JÁN HOLP','Ján Holp'),('Jána Holpa','Ján Holp'),
|
|
(' Jan Holp','Ján Holp'),('Holp Ján','Ján Holp'),('Holpa Jána','Ján Holp'),
|
|
('Holp, Jan','Ján Holp'),('Ondreja Megelu','Ondrej Megela'),
|
|
('Martina Šarišského','Martin Šarišský'),('Jána Malinovského','Ján Malinovský'),
|
|
('Ján Holpp','Ján Holp'),('Ján Hlop','Ján Holp'),('Ondrej Megla','Ondrej Megela'),
|
|
('Ján malinkvsý','Ján Malinovský'),('Ján Ptká','Ján Pták'),
|
|
])
|
|
def test_variants_and_bounded_typos(people,mention,name):
|
|
q=f'Aké práce má {mention} v roku 1999?'
|
|
before=copy.deepcopy(people)
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='ok' and [m['name'] for m in r['matches']]==[name]
|
|
assert r['resolved_query']==q[:q.index(mention.strip())]+name+q[q.index(mention.strip())+len(mention.strip()):]
|
|
assert 'v roku 1999?' in r['resolved_query'] and people==before
|
|
|
|
|
|
def test_multiple_people_and_all_non_name_conditions_survive(people):
|
|
q='Porovnaj Jána Holppa a Ondreja Megelu; nie diplomový projekt, iba práce z roku 2021 s tagom RAG.'
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='ok'
|
|
assert r['resolved_query']=='Porovnaj Ján Holp a Ondrej Megela; nie diplomový projekt, iba práce z roku 2021 s tagom RAG.'
|
|
assert len(r['notices'])==1 and 'Ján Holp' in r['notices'][0]
|
|
|
|
|
|
def test_plural_keyword_question_does_not_hide_a_misspelled_person(people):
|
|
q='Aké spoločné kľúčové slová majú Ján Ptká a Ján Malinovský?'
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='ok'
|
|
assert [m['name'] for m in r['matches']]==['Ján Pták','Ján Malinovský']
|
|
assert r['resolved_query']==q.replace('Ptká','Pták')
|
|
|
|
|
|
@pytest.mark.parametrize('instruction', [
|
|
'potom pri každej osobe príslušné práce s typom a názvom, ak je uvedený',
|
|
'potom pri každej osobe relevantné práce',
|
|
'potom pri každom študentovi jednotlivé práce',
|
|
'potom pre obe osoby príslušné práce',
|
|
'potom pri vybraných osobách stručné zhrnutie',
|
|
])
|
|
def test_presentation_instructions_are_not_additional_people(people, instruction):
|
|
q=('V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt? '
|
|
'Najprv uveď spoločné roky, '+instruction+'. Pridaj zdroje.')
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='ok'
|
|
assert [m['name'] for m in r['matches']]==['Ján Holp','Ondrej Megela']
|
|
assert r['resolved_query']==q and r['notices']==[]
|
|
|
|
|
|
@pytest.mark.parametrize('mention',['Xaver Neexistujúci','xaver neexistujuci'])
|
|
def test_generic_person_reference_does_not_hide_another_unknown_subject(people, mention):
|
|
q=f'Porovnaj Ján Holp a {mention}. Pri každej osobe uveď príslušné práce.'
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='no_answer' and r['reason']=='unknown_person'
|
|
assert r['mention']==mention
|
|
|
|
|
|
@pytest.mark.parametrize('q',[
|
|
'Aké práce má Xaver Neexistujúci?', 'Aké práce má xaver neexistujuci?',
|
|
'Aký je názov práce xaver neexistujuci?',
|
|
'Porovnaj práce Jána Holpa a Xavera Neexistujúceho.',
|
|
'Porovnaj xaver neexistujuci a jan holp.',
|
|
'Porovnaj jan holp a xaver neexistujuci.',
|
|
'Aké práce má Ján Hxxxx?', 'Aké práce má Xaver?',
|
|
'Z kandidátov Ján Holp, neznama osoba, Ondrej Megela vyber osoby s prácou v roku 2024.',
|
|
'Z kandidátov Ján Holp a neznama osoba vyber osoby s prácou v roku 2024.',
|
|
])
|
|
def test_unknown_name_cannot_turn_into_global_query(people,q):
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='no_answer' and r['reason']=='unknown_person' and not r['candidates']
|
|
session=FakeSession({gs.CATALOG_QUERY:people})
|
|
graph=gs.search_graph_with_session(session,q)
|
|
assert graph['status']=='no_answer' and graph['facts']==[]
|
|
assert all(statement==gs.CATALOG_QUERY for statement,_ in session.calls)
|
|
|
|
|
|
def test_nearby_catalog_names_require_selection_and_exact_name_wins():
|
|
catalog=[person('Martin Novák','novak'),person('Martin Novík','novik')]
|
|
r=resolve_person_names('Aké práce má Martin Novek?',catalog)
|
|
assert r['status']=='ambiguous'
|
|
assert {c['name'] for c in r['candidates']}=={'Martin Novák','Martin Novík'}
|
|
assert resolve_person_names('Aké práce má Martin Novák?',catalog)['matches'][0]['name']=='Martin Novák'
|
|
|
|
|
|
def test_short_first_name_is_not_automatically_assigned(people):
|
|
people = [*people, {'kind':'DocumentAuthor','name':'Ján Holp'}]
|
|
r=resolve_person_names('Aké práce má Ján?',people)
|
|
assert r['status']=='ambiguous' and len(r['candidates'])==3
|
|
assert all(c['name'].startswith('Ján ') for c in r['candidates'])
|
|
|
|
|
|
def test_full_exact_name_wins_over_another_person_inflection():
|
|
catalog=[person('Ján Novák','jan'),person('Jana Novák','jana')]
|
|
r=resolve_person_names('Aké práce má Jana Novák?',catalog)
|
|
assert r['matches'][0]['name']=='Jana Novák'
|
|
|
|
|
|
def test_competing_valid_inflections_require_selection():
|
|
catalog=[person('Ján Novák','jan'),person('Jana Novák','jana')]
|
|
r=resolve_person_names('Aký je názov práce Jana Novaka?',catalog)
|
|
assert r['status']=='ambiguous'
|
|
assert {c['name'] for c in r['candidates']}=={'Ján Novák','Jana Novák'}
|
|
|
|
|
|
def test_rag_evidence_uses_resolved_identity_not_another_name_inflection():
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
|
from tools.rag.rag_utils import build_context_text
|
|
catalog=[person('Ján Novák','jan'),person('Jana Novák','jana')]
|
|
q='Aký je názov diplomovej práce Jana Novák?'
|
|
sources=[{'source_id':f'S{i}','title':row['name'],'document_path':row['source_document'],
|
|
'source_url':f'https://example.test/{i}',
|
|
'text':'Sekcia: Diplomová práca 2024\nNázov: Výskum jazyka '+str(i)+'.'}
|
|
for i,row in enumerate(catalog,1)]
|
|
payload={'sources':sources,'context':build_context_text(sources),
|
|
'name_resolution':resolve_person_names(q,catalog)}
|
|
c=rag_evidence_context(payload['name_resolution']['resolved_query'],payload)
|
|
assert c['requirements']['people']==['Jana Novák']
|
|
assert {f['person'] for f in c['facts']}=={'Jana Novák'}
|
|
|
|
|
|
def test_canonicalized_question_keeps_natural_answer_grammar():
|
|
from tools.answer_evaluation.presentation import fact_sentences
|
|
claim={'person':'Ján Holp','work_type':'diploma_thesis','title':'Ukážkový výskum','year':2021}
|
|
text=fact_sentences(claim,'Aký je názov diplomovej práce Ján Holp?')
|
|
assert text.startswith('Ján Holp má diplomovú prácu s názvom') and '2021' in text
|
|
assert fact_sentences(claim,'Aký je názov diplomovej práce Jána Holpa?').startswith('Diplomová práca Jána Holpa')
|
|
|
|
|
|
def test_two_damaged_tokens_are_never_guessed(people):
|
|
assert resolve_person_names('Aké práce má Jaann Hlop?',people)['status']=='no_answer'
|
|
|
|
|
|
def test_document_author_is_not_implicitly_a_student(people):
|
|
catalog=[*people,{'kind':'DocumentAuthor','name':'Daniel Hladek'}]
|
|
r=resolve_person_names('Aké práce má Daniel Hladek?',catalog)
|
|
assert r['status']=='no_answer'
|
|
author=resolve_person_names('Ktoré dokumenty napísal autor Daniel Hladekk?',catalog)
|
|
assert author['status']=='ok' and author['matches'][0]['name']=='Daniel Hladek'
|
|
|
|
|
|
def test_duplicate_names_can_be_selected_by_actual_profile_path():
|
|
catalog=[person('Ján Novák','novak_a'),person('Ján Novák','novak_b')]
|
|
q='Aké práce má Ján Novák?'
|
|
assert resolve_person_names(q,catalog)['status']=='ambiguous'
|
|
selected=resolve_person_names(q+' Profil: '+catalog[1]['source_document'],catalog)
|
|
assert selected['status']=='ok' and selected['matches'][0]['source_document']==catalog[1]['source_document']
|
|
|
|
|
|
def test_multiword_name_supports_surname_first():
|
|
r=resolve_person_names('Aké práce má Chis Bogdan Paul?', [person('Bogdan Paul Chiș','bogdan')])
|
|
assert r['status']=='ok' and r['matches'][0]['name']=='Bogdan Paul Chiș'
|
|
|
|
|
|
def test_literal_nonstandard_catalog_name_preserves_numeric_suffix():
|
|
catalog=[person('Martin NaN1','nan')]
|
|
for mention in ('Martin NaN1','NaN1 Martin'):
|
|
r=resolve_person_names('Aké práce má '+mention+' v roku 1999?',catalog)
|
|
assert r['status']=='ok' and r['resolved_query']=='Aké práce má Martin NaN1 v roku 1999?'
|
|
assert resolve_person_names('Aké práce má Martin NaN2?',catalog)['status']=='no_answer'
|
|
|
|
|
|
@pytest.mark.parametrize('q',[
|
|
'Ktorí študenti majú tag RAG a diplomový projekt v roku 2026?',
|
|
'Aké práce boli v roku 2020?', 'Vyhľadaj práce s názvom „Ján Holpp“.',
|
|
'Ktorí študenti majú tag Ján Holpp?',
|
|
'Ktorí študenti majú kľúčové slovo Ján Holpp a prácu v roku 2026?',
|
|
])
|
|
def test_global_queries_and_quoted_titles_are_not_corrected_as_people(people,q):
|
|
r=resolve_person_names(q,people)
|
|
assert r['status']=='ok' and r['resolved_query']==q and not r['matches']
|
|
|
|
|
|
def test_malinovsky_sarissky_never_matches_martin_jancura(people):
|
|
r=resolve_person_names('Porovnaj Jána Malinovského a Martina Šarišského.',people)
|
|
assert {m['name'] for m in r['matches']}=={'Ján Malinovský','Martin Šarišský'}
|
|
|
|
|
|
@pytest.fixture
|
|
def names_db(tmp_path,people):
|
|
path=tmp_path/'index.sqlite'
|
|
with sqlite3.connect(path) as conn:
|
|
conn.execute('CREATE TABLE documents(path,title,published)')
|
|
for row in people:
|
|
conn.execute('INSERT INTO documents VALUES(?,?,?)',(row['source_document'],row['name'],1))
|
|
conn.execute('INSERT INTO documents VALUES(?,?,?)',('pages/students/2022/hidden/README.md','Xaver Skrytý',0))
|
|
return path
|
|
|
|
|
|
def test_catalog_honors_publication_policy(names_db):
|
|
assert not any(p['name']=='Xaver Skrytý' for p in sqlite_people(names_db))
|
|
assert any(p['name']=='Xaver Skrytý' for p in sqlite_people(names_db,published_only=False))
|
|
|
|
|
|
@pytest.mark.parametrize('q,status',[
|
|
('Aké práce má Xaver Neexistujúci?','no_answer'),('Aké práce má Ján?','ambiguous')])
|
|
def test_rag_does_not_search_or_generate_for_unresolved_subject(names_db,monkeypatch,q,status):
|
|
from tools.rag import rag_utils
|
|
from tools.answers.answer_delivery import deliver_answer
|
|
monkeypatch.setattr(rag_utils,'search_database',lambda *a,**kw:pytest.fail('Must not search globally'))
|
|
r=deliver_answer(q,engine='rag',db_file=names_db,complete=lambda _:pytest.fail('Must not call model'))
|
|
assert r['status']==status and r['accepted_for_delivery'] and r['source_urls']==[]
|
|
|
|
|
|
def test_graph_correction_retains_year_type_and_person_ids(people):
|
|
session=FakeSession({gs.CATALOG_QUERY:people})
|
|
q='Aké diplomové práce má Jan Holpp v roku 1999?'
|
|
r=gs.search_graph_with_session(session,q)
|
|
assert r['query']==q and 'Ján Holp' in r['resolved_query']
|
|
params=next(params for statement,params in session.calls if statement==gs.THESIS_FACTS_QUERY)
|
|
assert params['person_ids']==['jan_holp'] and params['years']==[1999]
|
|
assert r['constraints']['thesis_types']==['diploma_thesis']
|
|
|
|
|
|
def test_both_engines_use_same_resolution_and_notice(names_db,people,monkeypatch):
|
|
from tools.answers import answer_delivery as delivery
|
|
from tools.rag import rag_utils
|
|
question='Aké práce má Ján Holpp v roku 1999?'
|
|
resolution=resolve_person_names(question,people)
|
|
payload={'status':'ok','sources':[],'context':'','name_resolution':resolution,
|
|
'resolved_query':resolution['resolved_query'],'index_version':None}
|
|
checked={'status':'no_answer','reason':'no_matching_facts','answer':'Pre rok 1999 sa údaje nenašli.',
|
|
'source_urls':[],'claims':[],'accepted_for_delivery':True,'complete':True,
|
|
'truncated':False,'truncation_reasons':[]}
|
|
monkeypatch.setattr(rag_utils,'search_database',lambda *a,**k: {'results':[],'engine':'hybrid','strategies':[]})
|
|
monkeypatch.setattr(delivery,'build_rag_context',lambda *a,**k:payload)
|
|
monkeypatch.setattr(delivery,'expand_comparison_sources',lambda db,q,p:p)
|
|
monkeypatch.setattr(delivery,'prepare_named_content',lambda db,q,p:p)
|
|
monkeypatch.setattr(delivery,'compact_rag_payload',lambda p:p)
|
|
def rag(q,p,model):
|
|
assert q==resolution['resolved_query']
|
|
return dict(checked)
|
|
monkeypatch.setattr(delivery,'generate_verified_rag',rag)
|
|
rag_result=delivery.deliver_answer(question,engine='rag',db_file=names_db)
|
|
monkeypatch.setattr(delivery,'search_graph',lambda *a,**k:payload)
|
|
monkeypatch.setattr(delivery,'graph_context',lambda p:{'name_resolution':resolution})
|
|
def graph(q,context,prompt,model):
|
|
assert q==resolution['resolved_query']
|
|
return {'accepted_for_delivery':True,'answer':dict(checked),'attempts':[]}
|
|
monkeypatch.setattr(delivery,'generate_checked_answer',graph)
|
|
graph_result=delivery.deliver_answer(question,engine='graphrag',db_file=names_db)
|
|
assert rag_result['answer']==graph_result['answer']
|
|
assert rag_result['answer'].startswith('Rozpoznané meno: **Ján Holp**')
|