104 lines
6.1 KiB
Python
104 lines
6.1 KiB
Python
"""Real HTTP and MCP conversations exercise the shared delivery path.
|
|
|
|
The language provider and retrieval storage are fixtures; planning, academic
|
|
selection, scope checks, signed context, final audit and transport are real.
|
|
These are known regressions, not an independent accuracy measurement.
|
|
"""
|
|
import asyncio
|
|
from contextlib import nullcontext
|
|
import json
|
|
import os
|
|
import sys
|
|
import pytest
|
|
import app.routes as routes
|
|
from packages.claude import client
|
|
from test_claude_packages import backend,ROOT
|
|
from tools.answers import answer_delivery,question_plan
|
|
from tools.knowledge import query,store
|
|
from tools.knowledge.sources import profile_units
|
|
from tools.knowledge.academic import answer_academic
|
|
from tools.knowledge.planned_answers import answer_facets
|
|
from tools.sync import index_state
|
|
|
|
|
|
@pytest.fixture
|
|
def conversation_backend(backend,monkeypatch):
|
|
version={'version':'conversation-fixture'}
|
|
people=[{'kind':'Person','name':name,'source_document':f'pages/students/2022/{slug}/README.md'}
|
|
for name,slug in [('Eva Nová','eva_nova'),('Dana Iná','dana_ina')]]
|
|
units=profile_units({'title':'Eva Nová','path':people[0]['source_document'],'content':
|
|
'# Diplomová práca 2027\nStretnutie 5.5.2026\nStav:\n- Funguje model BERT.\nÚlohy:\n- Vyhodnoťte model BERT.\n'
|
|
'Stretnutie 1.4.2026\nStav:\n- Skúšaný model.\n# Diplomový projekt 2 2020\nStav:\n- Funguje databáza Cassandra.\n'
|
|
'# Diplomový projekt 2020\nStav:\n- Funguje indexovanie.'},include_records=True)
|
|
units+=profile_units({'title':'Dana Iná','path':people[1]['source_document'],'content':
|
|
'# Bakalárska práca 2026\nStretnutie 6.6.2025\nStav:\n- Funguje vyhľadávanie.\nÚlohy:\n- Pripravte experimenty.'},include_records=True)
|
|
index={'index_version':version,'version':'k','academic_schema':1,'units':units}
|
|
monkeypatch.setattr(answer_delivery,'index_snapshot',lambda *a,**k:nullcontext(version))
|
|
monkeypatch.setattr(index_state,'read_version',lambda *a,**k:version)
|
|
monkeypatch.setattr(question_plan,'planning_catalog',lambda *a,**k:people)
|
|
monkeypatch.setattr(store,'load_index',lambda *a,**k:index)
|
|
def model(messages):
|
|
p=json.loads(messages[-1]['content'])
|
|
if 'execution_plan' in p:
|
|
v={'plan_matches_request':True,'all_requirements_identified':True,'facts_supported':True,'clear':True,
|
|
'requirements':[{'description':p['resolved_question'],'question_segments':['Q1'],'status':'answered',
|
|
'answer_segments':[p['answer_segments'][0]['id']],'evidence_ids':[p['evidence'][0]['id']]}]}
|
|
elif 'claims' in p:
|
|
v={'coverage':'supported','verdicts':[{'claim_id':r['claim_id'],'decision':'supported','reason':'entailed'} for r in p['claims']]}
|
|
else:
|
|
v={'claims':[{'text':r['text'].lstrip('- '),'kind':'description','evidence_ids':[r['id']]} for r in p['evidence']]}
|
|
return {'choices':[{'message':{'content':json.dumps(v)}}]}
|
|
def retrieve(question,*,plan,engine,**kw):
|
|
return (answer_facets(plan,index,units,engine=engine,limit=8) if plan.spec.intent=='facets' else
|
|
answer_academic(question,index,engine=engine,mode=plan.spec.intent,plan=plan))
|
|
monkeypatch.setattr(query,'deliver_content',retrieve)
|
|
monkeypatch.setattr(routes,'deliver_answer',lambda q,**kw:answer_delivery.deliver_answer(q,complete=model,**kw))
|
|
return backend
|
|
|
|
|
|
STEPS=[
|
|
('rag','Kedy mala Eva Nová posledné stretnutie a o čom bolo?','ok',['Eva Nová','2026-05-05','BERT'],['Dana Iná','2026-04-01']),
|
|
('graphrag','Čo mala urobiť ďalej v danom stretnutí?','ok',['Eva Nová','2026-05-05','model BERT'],['Dana Iná','2026-04-01']),
|
|
('graphrag','Kedy mala Dana Iná posledné stretnutie a o čom bolo?','ok',['Dana Iná','2025-06-06'],['Eva Nová','2026-05-05']),
|
|
('rag','Na čom pracovala v danom stretnutí?','ok',['Dana Iná','2025-06-06'],['Eva Nová','2026-05-05']),
|
|
('rag','Aké metódy používala Eva Nová pri diplomovom projekte v roku 2020?','ambiguous',['Diplomový projekt'],['Dana Iná']),
|
|
('graphrag','Daj obe.','ok',['Cassandra','indexovanie'],['Dana Iná','BERT']),
|
|
('rag','Pri tej druhej práci.','ok',['Eva Nová'],['Dana Iná','BERT']),
|
|
('graphrag','Na čom pracovala Eva Nová na stretnutí 1.4.2026?','ok',['2026-04-01','Skúšaný model'],['2026-05-05','Dana Iná']),
|
|
]
|
|
|
|
|
|
def check(value,engine,q,status,required,forbidden):
|
|
assert value['query']==q and value['engine']==engine and value['status']==status,value
|
|
assert value['accepted_for_delivery']
|
|
assert all(s in value['answer'] for s in required),value
|
|
assert all(s not in value['answer'] for s in forbidden),value
|
|
assert value['delivery_check']['code'] in {'complete','partial','clarification_required'}
|
|
assert '_delivery_plan' not in value and '_delivery_facts' not in value
|
|
|
|
|
|
@pytest.mark.parametrize('transport',['http','mcp'])
|
|
def test_whole_conversation_switches_people_work_and_meeting_without_scope_leak(conversation_backend,transport):
|
|
async def run():
|
|
from mcp import ClientSession,StdioServerParameters
|
|
from mcp.client.stdio import stdio_client
|
|
params=StdioServerParameters(command=sys.executable,args=[str(ROOT/'packages/claude/mcp_server.py')],env=dict(os.environ))
|
|
async with stdio_client(params) as (read,write):
|
|
async with ClientSession(read,write) as session:
|
|
await session.initialize();token=None
|
|
for engine,q,status,required,forbidden in STEPS:
|
|
args={'query':q}
|
|
if token:args['context_token']=token
|
|
reply=await session.call_tool('zpwiki_'+engine,args)
|
|
assert not reply.isError,reply
|
|
value=reply.structuredContent
|
|
check(value,engine,q,status,required,forbidden)
|
|
token=(value.get('conversation_context') or {}).get('token')
|
|
if transport=='mcp':asyncio.run(run())
|
|
else:
|
|
token=None
|
|
for engine,q,status,required,forbidden in STEPS:
|
|
value=client.query_backend(q,engine,context_token=token)
|
|
check(value,engine,q,status,required,forbidden)
|
|
token=(value.get('conversation_context') or {}).get('token')
|