162 lines
8.6 KiB
Python
162 lines
8.6 KiB
Python
"""User-facing failure distinctions and real graph projection through delivery."""
|
|
import json
|
|
from contextlib import nullcontext
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
import app.main as main
|
|
import app.routes as routes
|
|
from packages.claude import client
|
|
from tools.answers import answer_delivery as delivery
|
|
from tools.answers.answer_errors import MESSAGES
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
|
from test_claude_packages import backend, KEY
|
|
|
|
QUESTION='Aké práce má Eva Nová?'
|
|
URL='https://example.test/eva'
|
|
|
|
|
|
def graph_session(size=4):
|
|
rows=[{'person':'Eva Nová','person_element_id':'eva','document_path':'eva.md',
|
|
'source_url':URL,'work_id':f'work:{i}','work_element_id':f'w{i}',
|
|
'work_type':'diploma_project','year':2024 if i<2 else 2025,
|
|
'heading':f'Diplomový projekt {i+1} '+('2024' if i<2 else '2025'),
|
|
'title':f'Téma projektu {i+1}'} for i in range(size)]
|
|
catalog=[{'kind':'Person','element_id':'eva','name':'Eva Nová','source_document':'eva.md'}]
|
|
catalog += [{'kind':'Work','element_id':f'w{i}','id':f'work:{i}',
|
|
'heading':r['heading'],'title':r['title']} for i,r in enumerate(rows)]
|
|
return FakeSession({gs.CATALOG_QUERY:catalog,gs.WORK_FACTS_QUERY:rows})
|
|
|
|
|
|
def select_graph(messages):
|
|
context=json.loads(messages[1]['content'])['graph_context']
|
|
fields=['person','record_id','work_type','work_label','title','year']
|
|
answer={'status':context['status'],'answer':'',
|
|
'claims':[{k:r[k] for k in fields if r.get(k) is not None} for r in context['facts']],
|
|
'source_urls':[s['source_url'] for s in context['sources'] if s.get('source_url')]}
|
|
return {'choices':[{'message':{'content':json.dumps(answer,ensure_ascii=False)}}]}
|
|
|
|
|
|
@pytest.mark.parametrize('limit',[1,2,4,5,20])
|
|
@pytest.mark.parametrize('size',[0,4])
|
|
def test_graph_limit_reaches_search_and_keeps_distinct_bound_records(monkeypatch,limit,size):
|
|
session=graph_session(size);calls=[]
|
|
def search(question,**kwargs):
|
|
calls.append((question,kwargs))
|
|
return gs.search_graph_with_session(session,question,**kwargs)
|
|
monkeypatch.setattr(delivery,'search_graph',search)
|
|
r=delivery.deliver_answer(QUESTION,engine='graphrag',limit=limit,complete=select_graph)
|
|
assert calls==[(QUESTION,{'limit':limit})]
|
|
assert r['accepted_for_delivery'] and len(r['claims'])==min(size,limit)
|
|
assert len({c['record_id'] for c in r['claims']})==min(size,limit)
|
|
for claim in r['claims']:
|
|
i=int(claim['record_id'].split(':')[-1])
|
|
assert claim['person']=='Eva Nová' and claim['title']==f'Téma projektu {i+1}'
|
|
assert claim['year']==(2024 if i<2 else 2025)
|
|
if size>limit:
|
|
assert r['complete'] is False and r['truncated'] is True
|
|
assert 'fact_limit' in r['truncation_reasons']
|
|
assert 'Výpis je neúplný.' in r['answer'] and 'Výpis môžeš zúžiť' in r['answer']
|
|
else:
|
|
assert r['complete'] is True and not r['truncated']
|
|
assert 'Výpis je neúplný.' not in r['answer']
|
|
|
|
|
|
def test_api_passes_graph_limit_and_hides_diagnostics(monkeypatch):
|
|
session=graph_session();calls=[]
|
|
def search(question,**kwargs):
|
|
calls.append(kwargs['limit'])
|
|
return gs.search_graph_with_session(session,question,**kwargs)
|
|
monkeypatch.setattr(delivery,'search_graph',search)
|
|
monkeypatch.setattr(delivery,'complete_with_configured_model',select_graph)
|
|
monkeypatch.setattr(main,'embed_query',lambda _:None)
|
|
with TestClient(main.app) as api:
|
|
response=api.post('/answer',json={'query':QUESTION,'engine':'graphrag','limit':1},
|
|
headers={'X-API-Key':KEY})
|
|
assert response.status_code==200
|
|
r=response.json()
|
|
assert calls==[1] and len(r['claims'])==1
|
|
assert r['truncated'] and 'Výpis môžeš zúžiť' in r['answer']
|
|
assert 'attempts' not in r and 'retrieval_context' not in r
|
|
assert api.post('/answer',json={'query':QUESTION,'engine':'graphrag','limit':0},
|
|
headers={'X-API-Key':KEY}).status_code==422
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
@pytest.mark.parametrize('returned_error',[False,True])
|
|
def test_failed_retrieval_is_not_no_answer_and_never_calls_model(monkeypatch,engine,returned_error):
|
|
def search(*args,**kwargs):
|
|
if returned_error:return {'status':'error','reason':'PRIVATE_RETRIEVAL_FAILURE'}
|
|
raise RuntimeError('PRIVATE_RETRIEVAL_FAILURE')
|
|
monkeypatch.setattr(delivery,'search_graph',search)
|
|
monkeypatch.setattr(delivery,'build_rag_context',search)
|
|
monkeypatch.setattr(delivery,'index_snapshot',lambda *a,**kw:nullcontext())
|
|
r=delivery.deliver_answer(QUESTION,engine=engine,
|
|
complete=lambda _:pytest.fail('No model after retrieval failure'))
|
|
assert r['status']=='error' and r['reason']=='retrieval_unavailable'
|
|
assert not r['accepted_for_delivery'] and r['answer']==MESSAGES['retrieval_unavailable']
|
|
assert 'PRIVATE' not in json.dumps(r)
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
@pytest.mark.parametrize('mode',['transport','invalid_format','unsupported_claim'])
|
|
def test_model_failure_is_distinct_from_missing_data(monkeypatch,mode,engine):
|
|
if engine=='graphrag':
|
|
monkeypatch.setattr(delivery,'search_graph',lambda q,**kw:gs.search_graph_with_session(graph_session(),q,**kw))
|
|
else:
|
|
from tools.rag.rag_utils import build_context_text
|
|
source={'title':'Eva Nová','source_url':URL,'document_path':'pages/students/2022/eva_nova/README.md',
|
|
'source_id':'S1','text':'Dokument: Eva Nová\nSekcia: Diplomová práca 2024\n\nNázov: Ukážkový systém.'}
|
|
payload={'status':'ok','complete':True,'truncated':False,'truncation_reasons':[],
|
|
'index_version':None,'sources':[source],'context':build_context_text([source])}
|
|
monkeypatch.setattr(delivery,'build_rag_context',lambda *a,**kw:payload)
|
|
monkeypatch.setattr(delivery,'index_snapshot',lambda *a,**kw:nullcontext())
|
|
monkeypatch.setattr(delivery,'expand_comparison_sources',lambda db,q,p:p)
|
|
monkeypatch.setattr(delivery,'prepare_named_content',lambda db,q,p:p)
|
|
calls=[]
|
|
def fail(messages):
|
|
calls.append(messages)
|
|
if mode=='transport':raise RuntimeError('PRIVATE_MODEL_FAILURE')
|
|
if mode=='invalid_format':text='PRIVATE NOT JSON'
|
|
else:text=json.dumps({'status':'ok','claims':[{'person':'PRIVATE WRONG PERSON','year':2099}],
|
|
'source_urls':[URL],'answer':''})
|
|
return {'choices':[{'message':{'content':text}}]}
|
|
r=delivery.deliver_answer(QUESTION,engine=engine,complete=fail)
|
|
reason='model_unavailable' if mode=='transport' else 'answer_rejected'
|
|
assert r['reason']==reason and r['status']=='error' and not r['accepted_for_delivery']
|
|
assert r['answer']==MESSAGES[reason] and 'spresni' not in r['answer'].casefold()
|
|
assert 'PRIVATE' not in r['answer'] and not r['claims'] and not r['source_urls']
|
|
assert len(calls)==(1 if mode=='transport' else 2)
|
|
|
|
|
|
@pytest.mark.parametrize('engine',['rag','graphrag'])
|
|
def test_busy_agent_is_not_an_ambiguous_question(monkeypatch,engine):
|
|
import threading
|
|
monkeypatch.setattr(delivery,'_capacity',threading.BoundedSemaphore(0))
|
|
r=delivery.deliver_answer(QUESTION,engine=engine,complete=lambda _:pytest.fail('Agent is busy'))
|
|
assert r['status']=='error' and r['reason']=='answer_capacity_exceeded'
|
|
assert r['answer']==MESSAGES['answer_capacity_exceeded'] and not r['accepted_for_delivery']
|
|
|
|
|
|
@pytest.mark.parametrize('reason',list(MESSAGES))
|
|
def test_client_and_api_have_consistent_safe_failure_wording(backend,reason):
|
|
backend['checked'].update(status='error',reason=reason,answer='PRIVATE REJECTED PROSE')
|
|
r=client.query_backend(QUESTION,'rag')
|
|
assert r['status']=='error' and r['reason']==reason
|
|
assert r['answer']==MESSAGES[reason] and not r['accepted_for_delivery']
|
|
assert 'PRIVATE' not in json.dumps(r)
|
|
|
|
|
|
@pytest.mark.parametrize('status,reason,text',[
|
|
('ambiguous','ambiguous_work','Ktorý projekt myslíš? Vyber projekt 1 alebo projekt 2.'),
|
|
('no_answer','requested_aspects_not_found','V podkladoch chýbajú údaje o riešených problémoch.'),
|
|
('ok',None,'Doložený výsledok. V podkladoch chýbajú údaje o experimentoch.'),
|
|
])
|
|
def test_clarification_missing_evidence_and_partial_answer_stay_distinct(backend,status,reason,text):
|
|
backend['checked'].update(status=status,reason=reason,answer=text)
|
|
r=client.answer_for_presentation(client.query_backend(QUESTION,'rag'))
|
|
assert r['status']==status and r['reason']==reason and r['accepted_for_delivery']
|
|
assert r['answer']==text
|