247 lines
14 KiB
Python
247 lines
14 KiB
Python
"""Multi-turn reference resolution with real source IDs, no paid model calls."""
|
|
import copy
|
|
import json
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from tools.answers.conversation import issue_context,resolve_question,seal,unseal,ContextError
|
|
from tools.answers.question_plan import build_question_plan,QuestionPlan,MeetingRequest,validate_model_plan,PlanRejected
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.academic import answer_academic
|
|
|
|
CATALOG=[{'name':'Jakub Schwarc','kind':'Person','source_document':'pages/students/2022/jakub_schwarc/README.md'},
|
|
{'name':'Ján Holp','kind':'Person','source_document':'pages/students/2016/jan_holp/README.md'}]
|
|
VERSION={'version':'v1'}
|
|
|
|
|
|
@pytest.fixture
|
|
def data(monkeypatch):
|
|
monkeypatch.setenv('SEARCH_API_KEY','context-fixture-only-key')
|
|
a=profile_units({'title':'Jakub Schwarc','path':CATALOG[0]['source_document'],'content':
|
|
'# Diplomová práca 2027\nStretnutie 5.11.2026\nStav:\n- Napísané skripty na vyhodnotenie.\n'
|
|
'Úlohy:\n- Spustite experiment na HPC.\nStretnutie 1.10.2026\nStav:\n- Porovnané modely.\n'
|
|
'# Diplomový projekt 2026\nStretnutie 10.6.2026\nStav:\n- Vyskúšaný starší model.\n'
|
|
'Stretnutie 27.2.\nStav:\n- Iná poznámka bez roku.'},include_records=True)
|
|
b=profile_units({'title':'Ján Holp','path':CATALOG[1]['source_document'],'content':
|
|
'# Diplomový projekt 2 2020\nStav:\n- Funguje indexovanie Cassandra.\n'
|
|
'# Diplomový projekt 2020\nStav:\n- Funguje databáza Cassandra.'},include_records=True)
|
|
return {'version':'content-v1','index_version':VERSION,'academic_schema':1,'units':a+b}
|
|
|
|
|
|
def latest(data):
|
|
q='Kedy mal Jakub Schwarc posledné stretnutie a o čom bolo?'
|
|
p=build_question_plan(q,CATALOG,index_version=VERSION)
|
|
result=answer_academic(q,data,engine='rag',mode='meeting',plan=p)
|
|
context=issue_context(q,result,CATALOG,data,VERSION)
|
|
assert len(context['meeting_options'])==1
|
|
return context
|
|
|
|
|
|
@pytest.mark.parametrize('q',[
|
|
'Na čom pracoval Jakub Schwarc v danom stretnutí?',
|
|
'ma com v danom stretnuti pracoval jakub schwarc',
|
|
'Na čom vtedy pracoval?',
|
|
])
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
def test_followup_keeps_exact_meeting_work_person_and_record_year(data,q,engine):
|
|
resolved=resolve_question(q,token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
assert not resolved.clarification
|
|
plan=build_question_plan(resolved.query,CATALOG,meeting=resolved.meeting,index_version=VERSION)
|
|
assert plan.spec.intent=='meeting' and plan.spec.period.start=='2026-11-05'
|
|
assert not any(2026 in w.years for w in plan.spec.works)
|
|
rows=[r for r in data['units'] if plan.spec.matches_evidence(r)]
|
|
reply=answer_academic(resolved.query,data,engine=engine,mode='meeting',plan=plan,units=rows)
|
|
assert reply['status']=='ok' and 'Napísané skripty' in reply['answer']
|
|
assert 'Porovnané modely' not in reply['answer'] and 'starší model' not in reply['answer']
|
|
assert '2026-10-01' not in reply['answer'] and 'Diplomový projekt 2026' not in reply['answer']
|
|
assert 'Dátum požadovaného stretnutia: 2026-11-05' in reply['answer']
|
|
|
|
|
|
@pytest.mark.parametrize('q',['Na čom vtedy pracoval?','Na čom pracoval Jakub Schwarc v danom stretnutí?','Daj obe.','Pri tej druhej práci.'])
|
|
def test_reference_without_context_asks_instead_of_global_search(data,q):
|
|
r=resolve_question(q,catalog=CATALOG,index=data,version=VERSION)
|
|
assert r.clarification and r.reason in {'missing_meeting_context','missing_followup_context'}
|
|
|
|
|
|
def test_another_person_cannot_borrow_a_meeting(data):
|
|
r=resolve_question('Na čom vtedy pracoval Ján Holp?',token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
assert r.reason=='context_subject_mismatch'
|
|
|
|
|
|
def test_fresh_standalone_question_does_not_inherit_old_subject_or_meeting(data):
|
|
q='Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?'
|
|
r=resolve_question(q,token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
assert r.query==q and r.meeting is None
|
|
|
|
|
|
def test_unknown_new_person_does_not_get_replaced_by_previous_person(data):
|
|
q='Aké práce má xaver neexistujuci?'
|
|
r=resolve_question(q,token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
assert r.query==q and 'Schwarc' not in r.query
|
|
|
|
|
|
@pytest.mark.parametrize('date',['1.10.2026','1. 10. 2026','2026-10-01'])
|
|
def test_date_spelling_variants_preserve_exact_same_record_scope(data,date):
|
|
p=build_question_plan('Na čom pracoval Jakub Schwarc na stretnutí '+date+'?',CATALOG)
|
|
assert p.spec.meeting.selection=='exact' and p.spec.period.start=='2026-10-01'
|
|
|
|
|
|
def test_implicit_new_aspect_still_belongs_to_previous_meeting(data):
|
|
r=resolve_question('A aké úlohy dostal?',token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
p=build_question_plan(r.query,CATALOG,meeting=r.meeting)
|
|
rows=[r for r in data['units'] if p.spec.matches_evidence(r)]
|
|
reply=answer_academic(r.query,data,engine='rag',mode='meeting',plan=p,units=rows)
|
|
assert 'Spustite experiment na HPC' in reply['answer'] and 'Napísané skripty' not in reply['answer']
|
|
|
|
|
|
@pytest.mark.parametrize('question',['Aké úlohy mal vtedy?','Čo mal urobiť ďalej v danom stretnutí?'])
|
|
def test_vtedy_after_a_predicate_is_not_an_unknown_person(data,question):
|
|
r=resolve_question(question,token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
assert not r.clarification and r.meeting.ids
|
|
p=build_question_plan(r.query,CATALOG,meeting=r.meeting)
|
|
assert not p.clarification and p.spec.facets==['plans']
|
|
|
|
|
|
def test_year_followup_changes_record_period_without_reusing_old_meeting_id(data):
|
|
r=resolve_question('A v roku 2025?',token=latest(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
p=build_question_plan(r.query,CATALOG)
|
|
assert not p.clarification and p.spec.intent=='meeting' and r.meeting is None
|
|
assert p.spec.meeting.selection=='latest' and p.spec.period.field=='record_date'
|
|
assert p.spec.period.start=='2025-01-01' and p.spec.period.end=='2025-12-31'
|
|
|
|
|
|
def test_staff_role_is_preserved_in_a_year_followup(data):
|
|
catalog=CATALOG+[{'kind':'DocumentAuthor','name':'Daniel Hladek'}]
|
|
q='Ktoré práce vedie Daniel Hladek?'
|
|
result={'status':'ok','accepted_for_delivery':True,'index_version':VERSION,'answer':'Výber vedených prác.'}
|
|
context=issue_context(q,result,catalog,data,VERSION)
|
|
r=resolve_question('A ktoré v roku 2026?',token=context['token'],catalog=catalog,index=data,version=VERSION)
|
|
p=build_question_plan(r.query,catalog)
|
|
assert p.spec.people[0].role=='supervisor' and p.spec.people[0].name=='Daniel Hladek'
|
|
assert p.spec.works[0].years==[2026]
|
|
|
|
|
|
def test_final_scope_check_rejects_true_facts_from_wrong_person_or_meeting(data):
|
|
from tools.answers.conversation import answer_matches_scope
|
|
old=next(r for r in data['units'] if r.get('record_date')=='2026-10-01' and r['kind']!='meeting')
|
|
wrong={'status':'ok','accepted_for_delivery':True,'evidence':[old],'claims':[]}
|
|
q='Na čom pracoval Jakub Schwarc na stretnutí 5.11.2026?'
|
|
assert not answer_matches_scope(q,wrong,CATALOG)
|
|
q='Aký je názov diplomovej práce Jána Holpa?'
|
|
assert not answer_matches_scope(q,wrong,CATALOG)
|
|
|
|
|
|
@pytest.mark.parametrize('change,reason',[
|
|
('signature','invalid_context'),('version','stale_context'),('expiry','expired_context'),('source','invalid_context')])
|
|
def test_invalid_expired_stale_and_missing_source_references_fail_closed(data,change,reason):
|
|
token=latest(data)['token'];version=VERSION
|
|
if change=='signature':token=token[:-3]+'xxx'
|
|
if change=='version':version={'version':'new-version'}
|
|
if change=='expiry':
|
|
payload=unseal(token,VERSION);payload['expires']=int(time.time())-1;token=seal(payload)
|
|
if change=='source':data['units']=[u for u in data['units'] if u['kind']!='meeting']
|
|
with pytest.raises(ContextError,match=reason):
|
|
resolve_question('Na čom vtedy pracoval?',token=token,catalog=CATALOG,index=data,version=version)
|
|
|
|
|
|
def test_multiple_previous_meetings_require_a_specific_selection(data):
|
|
context=latest(data);payload=unseal(context['token'],VERSION)
|
|
marker=next(r for r in data['units'] if r['kind']=='meeting' and r['date']=='2026-10-01')
|
|
payload['meetings'].append({'id':marker['id'],'person':marker['person'],'work_id':marker['work_id'],
|
|
'label':marker['work_label'],'date':marker['date'],'date_label':marker['date_label']})
|
|
r=resolve_question('Na čom vtedy pracoval?',token=seal(payload),catalog=CATALOG,index=data,version=VERSION)
|
|
assert r.reason=='ambiguous_meeting_reference' and '2026-11-05' in r.clarification and '2026-10-01' in r.clarification
|
|
|
|
|
|
def test_both_cannot_select_an_undisplayed_old_work_from_a_meeting_answer(data):
|
|
context=latest(data)
|
|
assert len(context['work_options'])==1
|
|
r=resolve_question('Daj obe.',token=context['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
assert r.reason=='ambiguous_work_reference'
|
|
|
|
|
|
def work_context(data):
|
|
q='Aké metódy používal Ján Holp pri diplomovom projekte v roku 2020?'
|
|
result={'status':'ambiguous','reason':'ambiguous_work','accepted_for_delivery':True,'index_version':VERSION,
|
|
'answer':'Ktorú prácu? Diplomový projekt 2 2020; Diplomový projekt 2020.'}
|
|
return issue_context(q,result,CATALOG,data,VERSION)
|
|
|
|
|
|
def test_both_reconstructs_one_complete_question_with_original_aspects(data):
|
|
r=resolve_question('Daj obe.',token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
p=build_question_plan(r.query,CATALOG)
|
|
assert not p.clarification and p.spec.facets==['methods']
|
|
assert p.spec.scope=='selected_works' and len(p.spec.works[0].labels)==2 and p.spec.works[0].years==[2020]
|
|
|
|
|
|
def test_second_work_selection_keeps_original_request_but_only_selected_work(data):
|
|
r=resolve_question('Pri tej druhej práci.',token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
p=build_question_plan(r.query,CATALOG)
|
|
assert not r.clarification and p.spec.facets==['methods']
|
|
assert p.spec.matches_work('Ján Holp','Diplomový projekt 2020','diploma_project',2020)
|
|
assert not p.spec.matches_work('Ján Holp','Diplomový projekt 2 2020','diploma_project',2020)
|
|
|
|
|
|
def test_second_selection_can_follow_a_successful_both_answer(data):
|
|
from tools.knowledge.planned_answers import answer_facets
|
|
both=resolve_question('Daj obe.',token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
plan=build_question_plan(both.query,CATALOG)
|
|
result=answer_facets(plan,data,data['units'],engine='rag',limit=5)
|
|
assert result['status']=='ok'
|
|
context=issue_context(both.query,result,CATALOG,data,VERSION)
|
|
second=resolve_question('Pri tej druhej práci.',token=context['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
plan=build_question_plan(second.query,CATALOG)
|
|
result=answer_facets(plan,data,data['units'],engine='rag',limit=5)
|
|
assert result['status']=='ok' and len({r['work_id'] for r in result['evidence']})==1
|
|
assert all(r['work_label']==context['work_options'][1]['label'] for r in result['evidence'])
|
|
|
|
|
|
def test_new_aspect_inherits_person_work_type_and_year(data):
|
|
r=resolve_question('A aké výsledky?',token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION)
|
|
p=build_question_plan(r.query,CATALOG)
|
|
assert p.spec.facets==['results'] and p.spec.people[0].name=='Ján Holp'
|
|
assert p.spec.works[0].work_type=='diploma_project' and p.spec.works[0].years==[2020]
|
|
|
|
|
|
def test_exact_date_is_not_silently_replaced_by_latest(data):
|
|
q='Na čom pracoval Jakub Schwarc na stretnutí 1.10.2026?'
|
|
p=build_question_plan(q,CATALOG)
|
|
assert p.spec.intent=='meeting' and p.spec.meeting.selection=='exact'
|
|
assert p.spec.period.start==p.spec.period.end=='2026-10-01'
|
|
rows=[r for r in data['units'] if r.get('record_date')=='2026-10-01']
|
|
reply=answer_academic(q,data,engine='rag',mode='meeting',plan=p,units=rows)
|
|
assert 'Porovnané modely' in reply['answer'] and 'Napísané skripty' not in reply['answer']
|
|
|
|
|
|
def test_model_cannot_drop_meeting_or_turn_proposal_into_metadata(data):
|
|
p=build_question_plan('Na čom pracoval Jakub Schwarc na stretnutí 1.10.2026?',CATALOG).spec
|
|
with pytest.raises(PlanRejected,match='meeting'):
|
|
validate_model_plan(p.model_copy(update={'meeting':None}),p)
|
|
p=build_question_plan('Čo by sa dalo spojiť z práce Jána Holpa a Jakuba Schwarca?',CATALOG).spec
|
|
assert p.intent=='proposal' and p.answer_mode=='proposal'
|
|
with pytest.raises(PlanRejected,match='answer mode'):
|
|
validate_model_plan(p.model_copy(update={'answer_mode':'fact'}),p)
|
|
|
|
|
|
def test_same_meaning_typo_and_paraphrase_use_same_route(data):
|
|
from tools.answers.conversation import normalize_question
|
|
for q in ['na com pracoval Jakub Schwarc','ma com pracoval Jakub Schwarc']:
|
|
p=build_question_plan(normalize_question(q),CATALOG)
|
|
assert p.spec.intent=='progress' and p.spec.answer_mode=='description'
|
|
|
|
|
|
def test_client_transports_context_without_rewriting_query(monkeypatch):
|
|
from packages.claude import client
|
|
captured={}
|
|
def request(config,endpoint,body):
|
|
captured.update(body)
|
|
return {'status':'ambiguous','reason':'test','query':body['query'],'engine':'rag',
|
|
'accepted_for_delivery':True,'answer':'Vyber stretnutie.','claims':[],'source_urls':[],
|
|
'complete':False,'truncated':False,'truncation_reasons':[],'index_version':VERSION,
|
|
'resolved_query':'Overená úplná otázka.','conversation_context':{'token':'opaque','work_options':[],'meeting_options':[]}}
|
|
monkeypatch.setattr(client,'request_json',request)
|
|
result=client.query_backend('Daj obe.','rag',config=client.Config('http://localhost:8000','test'),context_token='opaque')
|
|
assert captured['query']=='Daj obe.' and captured['context_token']=='opaque'
|
|
assert client.answer_for_presentation(result)['conversation_context']['token']=='opaque'
|