72 lines
4.2 KiB
Python
72 lines
4.2 KiB
Python
"""Source identity survives presentation field selection in both retrievers."""
|
|
import copy
|
|
import pytest
|
|
from tools.answers import delivery_contract as contract
|
|
from tools.answers.graph_context import delivery_facts
|
|
from tools.answers.question_plan import QuestionPlan, PersonRequest, WorkRequest
|
|
from tools.answers.verified_rag import generate_verified_rag
|
|
|
|
|
|
URL='https://example.test/eva'
|
|
|
|
|
|
def plan():
|
|
return QuestionPlan(question='Aký typ a rok má záznam „Diplomová práca 2027“ osoby Eva Nová?',
|
|
people=[PersonRequest(name='Eva Nová',source_document='pages/students/2020/eva/README.md')],
|
|
works=[WorkRequest(person='Eva Nová',work_type='diploma_thesis',years=[2027],labels=['diplomova praca 2027'])],
|
|
fields=['work_type','year'],scope='single_work')
|
|
|
|
|
|
def output(claims):
|
|
return {'status':'ok','accepted_for_delivery':True,'answer':'Eva Nová: diplomová práca, rok 2027.\nZdroj: '+URL,
|
|
'claims':claims,'source_urls':[URL],'evidence':[]}
|
|
|
|
|
|
def test_graph_source_scope_is_preserved_without_adding_unrequested_fields_to_prompt():
|
|
fact={'person':'Eva Nová','work_type':'diploma_thesis','year':2027,'source_url':URL}
|
|
context={'facts':[fact]};before=copy.deepcopy(context)
|
|
raw={'facts':[{'person':'Eva Nová','thesis_type':'diploma_thesis','year':2027,
|
|
'heading':'Diplomová práca 2027','thesis_id':'work-1','source_url':URL,'fact_type':'person_thesis'}]}
|
|
p=plan();result=output([{'person':'Eva Nová','work_type':'diploma_thesis','year':2027}])
|
|
assert not contract.enforce(p.question,'graphrag',p,result,facts=context['facts'])['accepted_for_delivery']
|
|
bindings=delivery_facts(context,raw)
|
|
assert contract.enforce(p.question,'graphrag',p,result,facts=bindings)['accepted_for_delivery']
|
|
assert context==before and bindings[-1]['record_id']=='work-1'
|
|
|
|
|
|
def test_person_tag_requires_separate_source_backed_work_for_a_conjunction():
|
|
p=plan();p.question='Ktorí študenti majú tag NLP a diplomovú prácu z roku 2027?'
|
|
p.fields=['keyword'];p.presentation='list'
|
|
fact={'person':'Eva Nová','keyword':'NLP','source_url':URL}
|
|
result=output([{'person':'Eva Nová','keyword':'NLP'}])
|
|
result['answer']='Eva Nová má tag NLP.\nZdroj: '+URL
|
|
assert not contract.enforce(p.question,'graphrag',p,result,facts=[fact])['accepted_for_delivery']
|
|
work={'person':'Eva Nová','work_type':'diploma_thesis','year':2027,'work_label':'Diplomová práca 2027','source_url':URL}
|
|
assert contract.enforce(p.question,'graphrag',p,result,facts=[fact,work])['accepted_for_delivery']
|
|
for change in [{'year':2024},{'source_url':'https://example.test/other'},{'person':'Dana Iná'}]:
|
|
assert not contract.enforce(p.question,'graphrag',p,result,facts=[fact,{**work,**change}])['accepted_for_delivery']
|
|
|
|
|
|
def test_rag_binds_actual_prepared_facts_instead_of_incomplete_model_claims():
|
|
p=plan()
|
|
payload={'status':'ok','reason':None,'index_version':{'version':'fixture'},'complete':True,'truncated':False,
|
|
'truncation_reasons':[],'named_scope_complete':True,'question_plan':p.model_dump(),
|
|
'sources':[{'source_id':'S1','title':'Eva Nová','document_path':'pages/students/2020/eva/README.md',
|
|
'source_url':URL,'categories':['dp2027'],'text':'# Diplomová práca 2027\nNázov: Pokusný systém.\n'}]}
|
|
from tools.rag.rag_utils import build_context_text
|
|
payload['context']=build_context_text(payload['sources'])
|
|
def forbidden(_):pytest.fail('Exact metadata needs no language model')
|
|
plain=generate_verified_rag(p.question,payload,forbidden)
|
|
assert '_delivery_facts' not in plain
|
|
result=generate_verified_rag(p.question,payload,forbidden,include_bindings=True)
|
|
facts=result.pop('_delivery_facts')
|
|
assert any(f.get('work_label')=='Diplomová práca 2027' for f in facts)
|
|
assert contract.enforce(p.question,'rag',p,result,facts=facts)['accepted_for_delivery']
|
|
|
|
|
|
def test_rejected_verification_is_not_diagnosed_as_provider_unavailability():
|
|
rejected=contract.diagnose({'status':'error','reason':'answer_rejected'},'q','rag')
|
|
unavailable=contract.diagnose({'status':'error','reason':'verification_unavailable'},'q','rag')
|
|
assert rejected['delivery_check']['code']=='request_not_fulfilled'
|
|
assert unavailable['delivery_check']['code']=='unavailable'
|