347 lines
18 KiB
Python
347 lines
18 KiB
Python
"""Exercise extracted skills, real HTTP boundaries and MCP stdio without a paid LLM."""
|
|
import asyncio
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from neo4j.exceptions import ServiceUnavailable
|
|
|
|
import app.main as main
|
|
import app.routes as routes
|
|
from scripts.build_claude_packages import build_package
|
|
from packages.claude import client
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
KEY = 'a' * 64
|
|
|
|
|
|
@pytest.fixture
|
|
def backend(monkeypatch):
|
|
monkeypatch.setattr(main, 'embed_query', lambda _: None)
|
|
evidence = {'status': 'ok', 'reason': None, 'complete': False, 'truncated': True,
|
|
'truncation_reasons': ['result_limit'], 'index_version': {'version': 'fixture'},
|
|
'sources': [{'source_id': 'S1', 'title': 'Eva Nová', 'document_path': 'pages/eva/README.md',
|
|
'source_url': 'https://example.test/eva'}],
|
|
'context': 'Eva Nová: Diplomová práca 2024.',
|
|
'facts': [{'person': 'Eva Nová', 'year': 2024}], 'paths': []}
|
|
checked = {k: copy.deepcopy(v) for k, v in evidence.items() if k not in {'sources', 'context', 'facts', 'paths'}}
|
|
checked.update(accepted_for_delivery=True, answer='Eva Nová — rok práce: 2024.\nVýpis je neúplný.',
|
|
claims=[{'person': 'Eva Nová', 'year': 2024}], source_urls=['https://example.test/eva'],
|
|
attempts=[{'raw_response': 'PRIVATE MODEL DIAGNOSTICS'}])
|
|
monkeypatch.setattr(routes, 'deliver_answer', lambda *args, **kw: copy.deepcopy(checked))
|
|
monkeypatch.setattr(routes, 'search_graph', lambda *args, **kw: copy.deepcopy(evidence))
|
|
monkeypatch.setattr(routes, 'build_rag_context', lambda *args, **kw: {
|
|
**{k: v for k, v in evidence.items() if k in routes.RagResponse.model_fields},
|
|
'query': args[1], 'engine': 'hybrid_fts5_embeddings', 'source_count': 1,
|
|
'strategies': [], 'instructions': [], 'answer_format': {}})
|
|
monkeypatch.setattr(routes, 'compact_rag_payload', lambda value, **kw: value)
|
|
state = {'raw': None, 'code': 200, 'requests': [], 'checked': checked, 'evidence': evidence}
|
|
with TestClient(main.app) as api:
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
value = {'ready': True, 'index_version': {'version': 'fixture'}, 'tools': {
|
|
engine: {'ready': True, 'index_version': {'version': 'fixture'}, 'reason': None}
|
|
for engine in ['rag', 'graphrag']}}
|
|
self.send_response(200)
|
|
self.end_headers()
|
|
self.wfile.write(json.dumps(value).encode())
|
|
|
|
def do_POST(self):
|
|
body = self.rfile.read(int(self.headers.get('Content-Length', '0')))
|
|
state['requests'].append((self.path, json.loads(body), self.headers.get('Authorization')))
|
|
if state['raw'] is None:
|
|
response = api.post(self.path, content=body, headers=dict(self.headers))
|
|
code, raw = response.status_code, response.content
|
|
else:
|
|
code, raw = state['code'], state['raw']
|
|
self.send_response(code)
|
|
if code == 302:
|
|
self.send_header('Location', state['url'] + '/credential-leak')
|
|
self.end_headers()
|
|
self.wfile.write(raw)
|
|
|
|
server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
state['url'] = f'http://127.0.0.1:{server.server_port}'
|
|
monkeypatch.setenv('ZP_TEST_HTTP_PORT', str(server.server_port))
|
|
monkeypatch.setenv('ZP_AGENT_URL', state['url'])
|
|
monkeypatch.setenv('ZP_AGENT_API_KEY', KEY)
|
|
monkeypatch.delenv('ZP_AGENT_CONFIG', raising=False)
|
|
try:
|
|
yield state
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join()
|
|
|
|
|
|
@pytest.mark.parametrize('engine', ['rag', 'graphrag'])
|
|
def test_answer_is_exact_checked_text_without_diagnostics(backend, engine):
|
|
result = client.query_backend('Práca Evy Novej?', engine)
|
|
assert result['accepted_for_delivery'] is True
|
|
assert result['answer'] == backend['checked']['answer']
|
|
assert result['complete'] is False and result['truncated'] is True
|
|
assert result['index_version'] == {'version': 'fixture'}
|
|
assert 'attempts' not in result and 'PRIVATE' not in json.dumps(result)
|
|
path, body, authorization = backend['requests'][0]
|
|
assert path == '/answer' and body['engine'] == engine and body['include_diagnostics'] is False
|
|
assert authorization == 'Bearer ' + KEY
|
|
|
|
|
|
@pytest.mark.parametrize('engine', ['rag', 'graphrag'])
|
|
def test_retrieval_explicitly_requires_final_verification(backend, engine):
|
|
result = client.query_backend('Práca Evy Novej?', engine, mode='retrieve')
|
|
assert result['status'] == 'ok'
|
|
assert result['requires_final_answer_verification'] is True
|
|
assert result['sources'] and 'answer' not in result
|
|
assert backend['requests'][0][1]['published_only'] is True
|
|
|
|
|
|
@pytest.mark.parametrize('code,reason', [(401, 'unauthorized'), (403, 'forbidden'),
|
|
(503, 'backend_unavailable'), (302, 'redirect_not_allowed')])
|
|
def test_transport_errors_and_redirects_do_not_disclose_payload(backend, code, reason):
|
|
backend.update(raw=b'SECRET ERROR', code=code)
|
|
value = client.query_backend('Práca?', 'rag')
|
|
assert value['reason'] == reason and value['answer'] == ''
|
|
assert len(backend['requests']) == 1 and 'SECRET' not in json.dumps(value)
|
|
|
|
|
|
@pytest.mark.parametrize('change', [
|
|
{'accepted_for_delivery': False}, {'accepted_for_delivery': 'true'},
|
|
{'complete': True}, {'truncation_reasons': 'limit'}, {'status': 'unexpected'},
|
|
{'answer': ''}, {'source_urls': 'https://example.test'}, {'claims': None}])
|
|
def test_rejected_or_malformed_answer_is_never_delivered(backend, change):
|
|
backend.update(raw=json.dumps({**backend['checked'], **change}).encode())
|
|
value = client.query_backend('Práca?', 'rag')
|
|
assert value['status'] == 'error' and not value['accepted_for_delivery']
|
|
assert value['answer'] == '' and 'Eva' not in json.dumps(value)
|
|
|
|
|
|
def test_invalid_json_and_oversized_response(backend):
|
|
for raw, reason in [(b'not json', 'invalid_backend_response'),
|
|
(b'x' * (client.MAX_RESPONSE_BYTES + 1), 'response_too_large')]:
|
|
backend['raw'] = raw
|
|
assert client.query_backend('Práca?', 'rag')['reason'] == reason
|
|
|
|
|
|
def test_actual_backend_denies_wrong_key_and_health_keeps_versions(backend):
|
|
invalid = client.Config(backend['url'], 'wrong-test-key')
|
|
assert client.query_backend('Práca?', 'rag', config=invalid)['reason'] == 'unauthorized'
|
|
health = client.backend_health()
|
|
assert health['ready'] and all(value['ready'] for value in health['tools'].values())
|
|
assert health['tools']['rag']['index_version'] == health['tools']['graphrag']['index_version']
|
|
|
|
|
|
@pytest.mark.parametrize('status', ['no_answer', 'ambiguous'])
|
|
def test_negative_and_ambiguous_answers_stay_distinct(backend, status):
|
|
backend['checked'].update(status=status, answer='Žiadne doložené údaje.' if status == 'no_answer' else 'Spresni meno.',
|
|
claims=[], source_urls=[])
|
|
result = client.query_backend('Práca?', 'rag')
|
|
assert result['status'] == status and result['accepted_for_delivery'] is True
|
|
|
|
|
|
@pytest.mark.parametrize('query', ['', ' ', 'x' * 501, None])
|
|
def test_invalid_query_does_not_reach_backend(backend, query):
|
|
assert client.query_backend(query, 'rag')['reason'] == 'invalid_query'
|
|
assert backend['requests'] == []
|
|
|
|
|
|
@pytest.mark.parametrize('url', ['http://remote.test', 'https://user:password@example.test',
|
|
'https://example.test/?key=secret', 'https://example.test/#x', 'file:///tmp/x'])
|
|
def test_unsafe_configuration_is_rejected_without_network(url):
|
|
with pytest.raises(client.ClientError):
|
|
client.Config(url, KEY).validate()
|
|
|
|
|
|
def test_config_file_permissions_and_environment_override(tmp_path, monkeypatch):
|
|
path = tmp_path / 'config.json'
|
|
path.write_text(json.dumps({'backend_url': 'https://example.test', 'api_key': KEY}))
|
|
path.chmod(0o644)
|
|
monkeypatch.setenv('ZP_AGENT_CONFIG', str(path))
|
|
monkeypatch.delenv('ZP_AGENT_URL', raising=False)
|
|
monkeypatch.delenv('ZP_AGENT_API_KEY', raising=False)
|
|
with pytest.raises(client.ClientError, match='0600'):
|
|
client.Config.load()
|
|
path.chmod(0o600)
|
|
assert client.Config.load().key == KEY
|
|
monkeypatch.setenv('ZP_AGENT_URL', 'http://localhost:8000')
|
|
assert client.Config.load().url == 'http://localhost:8000'
|
|
|
|
|
|
@pytest.mark.parametrize('engine', ['rag', 'graphrag'])
|
|
def test_zip_is_reproducible_and_runs_without_repository(tmp_path, backend, engine):
|
|
one = build_package(engine, tmp_path / 'one')
|
|
two = build_package(engine, tmp_path / 'two')
|
|
assert one['sha256'] == two['sha256']
|
|
archive_path = tmp_path / 'one' / one['archive']
|
|
folder = tmp_path / 'installed'
|
|
with zipfile.ZipFile(archive_path) as archive:
|
|
assert len(archive.namelist()) == 4
|
|
manifest = json.loads(archive.read(f'zpwiki-{engine}/manifest.json'))
|
|
for name, digest in manifest['files'].items():
|
|
assert hashlib.sha256(archive.read(f'zpwiki-{engine}/' + name)).hexdigest() == digest
|
|
archive.extractall(folder)
|
|
environment = {**os.environ, 'PYTHONPATH': '', 'PYTHONDONTWRITEBYTECODE': '1'}
|
|
command = [sys.executable, str(folder / f'zpwiki-{engine}/scripts/query.py'), 'Práca Evy Novej?']
|
|
result = subprocess.run(command, cwd=folder, env=environment, capture_output=True, text=True, timeout=15)
|
|
assert result.returncode == 0, result.stderr
|
|
assert json.loads(result.stdout)['answer'] == backend['checked']['answer']
|
|
|
|
|
|
def test_committed_archives_match_current_sources(tmp_path):
|
|
manifest = json.loads((ROOT / 'packages/claude/dist/packages.json').read_text())
|
|
for row in manifest['packages']:
|
|
engine = row['name'].removeprefix('zpwiki-')
|
|
rebuilt = build_package(engine, tmp_path)
|
|
actual = ROOT / 'packages/claude/dist' / row['archive']
|
|
assert hashlib.sha256(actual.read_bytes()).hexdigest() == rebuilt['sha256'] == row['sha256']
|
|
|
|
|
|
def test_mcp_stdio_initialization_discovery_call_and_rejection(backend):
|
|
from mcp import ClientSession, StdioServerParameters
|
|
from mcp.client.stdio import stdio_client
|
|
|
|
async def run():
|
|
params = StdioServerParameters(command=sys.executable,
|
|
args=[str(ROOT / 'packages/claude/mcp_server.py')],
|
|
env={**os.environ, 'PYTHONDONTWRITEBYTECODE': '1'})
|
|
async with stdio_client(params) as (read, write):
|
|
async with ClientSession(read, write) as session:
|
|
initialized = await session.initialize()
|
|
assert initialized.serverInfo.name == 'ZP Wiki'
|
|
listed = await session.list_tools()
|
|
assert {row.name for row in listed.tools} == {'zpwiki_rag', 'zpwiki_graphrag', 'zpwiki_health'}
|
|
for engine in ['rag', 'graphrag']:
|
|
result = await session.call_tool('zpwiki_' + engine, {'query': 'Práca Evy Novej?'})
|
|
assert not result.isError
|
|
assert result.structuredContent['answer'] == backend['checked']['answer']
|
|
rejected = await session.call_tool('zpwiki_rag', {'query': ''})
|
|
assert rejected.isError and rejected.structuredContent['answer'] == ''
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize('provider', ['anthropic', 'openai_compatible'])
|
|
def test_model_adapter_request_and_response_shapes(monkeypatch, provider):
|
|
from tools.answers.model_client import complete_with_configured_model
|
|
calls = []
|
|
monkeypatch.setenv('ANSWER_PROVIDER', provider)
|
|
monkeypatch.setenv('ANSWER_API_KEY', 'test-secret')
|
|
monkeypatch.setenv('ANSWER_MODEL', 'explicit-test-model')
|
|
monkeypatch.setenv('ANSWER_LLM_URL', 'https://model.example.test/api')
|
|
content = '{"status":"no_answer"}'
|
|
payload = {'stop_reason': 'end_turn', 'content': [{'type': 'text', 'text': content}]} if provider == 'anthropic' else {
|
|
'choices': [{'message': {'content': content}}]}
|
|
|
|
class Opener:
|
|
def open(self, request, **kw):
|
|
calls.append((request, json.loads(request.data)))
|
|
return io.BytesIO(json.dumps(payload).encode())
|
|
monkeypatch.setattr(urllib.request, 'build_opener', lambda *args: Opener())
|
|
value = complete_with_configured_model([{'role': 'system', 'content': 'Rules'},
|
|
{'role': 'user', 'content': 'Question'}])
|
|
assert value['choices'][0]['message']['content'] == content
|
|
request, body = calls[0]
|
|
assert body['model'] == 'explicit-test-model'
|
|
if provider == 'anthropic':
|
|
assert body['system'] == 'Rules' and body['messages'][0]['role'] == 'user'
|
|
assert request.get_header('X-api-key') == 'test-secret'
|
|
else:
|
|
assert body['messages'][0]['role'] == 'system'
|
|
assert request.get_header('Authorization') == 'Bearer test-secret'
|
|
|
|
|
|
def test_anthropic_truncated_output_is_rejected(monkeypatch):
|
|
from tools.answers.model_client import complete_with_configured_model
|
|
monkeypatch.setenv('ANSWER_PROVIDER', 'anthropic')
|
|
monkeypatch.setenv('ANSWER_API_KEY', KEY)
|
|
monkeypatch.setenv('ANSWER_MODEL', 'explicit-test-model')
|
|
monkeypatch.delenv('ANSWER_LLM_URL', raising=False)
|
|
class Opener:
|
|
def open(self, *args, **kwargs):
|
|
return io.BytesIO(b'{"stop_reason":"max_tokens","content":[]}')
|
|
monkeypatch.setattr(urllib.request, 'build_opener', lambda *args: Opener())
|
|
with pytest.raises(RuntimeError, match='Incomplete'):
|
|
complete_with_configured_model([{'role': 'user', 'content': 'Question'}])
|
|
|
|
|
|
def test_remote_mcp_authentication_and_full_protocol(backend, monkeypatch):
|
|
import httpx
|
|
from mcp import ClientSession
|
|
from mcp.client.streamable_http import streamable_http_client
|
|
from packages.claude.mcp_server import create_http_app
|
|
token = 'm' * 64
|
|
monkeypatch.setenv('ZP_MCP_ACCESS_TOKEN', token)
|
|
monkeypatch.setenv('ZP_MCP_PUBLIC_URL', 'https://mcp.example.test/mcp')
|
|
app = create_http_app()
|
|
|
|
async def run():
|
|
async with app.app.router.lifespan_context(app.app):
|
|
transport = httpx.ASGITransport(app=app)
|
|
async with httpx.AsyncClient(transport=transport, base_url='https://mcp.example.test') as http:
|
|
for headers in [{}, {'Authorization': 'Bearer wrong'}, {'Authorization': 'Bearer ' + KEY}]:
|
|
denied = await http.post('/mcp', json={}, headers=headers)
|
|
assert denied.status_code == 401 and denied.json() == {'error': 'unauthorized'}
|
|
async with httpx.AsyncClient(transport=transport, base_url='https://mcp.example.test',
|
|
headers={'Authorization': 'Bearer ' + token}) as http:
|
|
hostile = await http.post('/mcp', json={}, headers={'Host': 'untrusted.example.test',
|
|
'Accept': 'application/json, text/event-stream'})
|
|
assert hostile.status_code == 421
|
|
async with streamable_http_client('https://mcp.example.test/mcp', http_client=http) as (read, write, _):
|
|
async with ClientSession(read, write) as session:
|
|
await session.initialize()
|
|
assert len((await session.list_tools()).tools) == 3
|
|
for engine in ['rag', 'graphrag']:
|
|
result = await session.call_tool('zpwiki_' + engine, {'query': 'Práca Evy Novej?'})
|
|
assert not result.isError
|
|
assert result.structuredContent['answer'] == backend['checked']['answer']
|
|
health = await session.call_tool('zpwiki_health')
|
|
assert not health.isError and health.structuredContent['ready']
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize('token,url', [('', 'https://mcp.example.test/mcp'),
|
|
('short', 'https://mcp.example.test/mcp'), (KEY, 'https://mcp.example.test/mcp'),
|
|
('m' * 64, 'http://mcp.example.test/mcp'), ('m' * 64, 'https://mcp.example.test/other')])
|
|
def test_remote_mcp_refuses_insecure_startup(backend, monkeypatch, token, url):
|
|
from packages.claude.mcp_server import create_http_app
|
|
monkeypatch.setenv('ZP_MCP_ACCESS_TOKEN', token)
|
|
monkeypatch.setenv('ZP_MCP_PUBLIC_URL', url)
|
|
with pytest.raises(ValueError):
|
|
create_http_app()
|
|
|
|
|
|
def test_graph_serialization_errors_have_shared_contract(monkeypatch):
|
|
monkeypatch.setattr(routes, 'search_graph', lambda *args, **kwargs: {'invalid': object()})
|
|
with TestClient(main.app) as client:
|
|
response = client.post('/graphrag', json={'query': 'Práca?'}, headers={'X-API-Key': KEY})
|
|
assert response.status_code == 500 and response.json()['reason'] == 'internal_error'
|
|
|
|
|
|
def test_graph_api_uses_shared_error_contract_and_validates_query(monkeypatch):
|
|
def unavailable(*args, **kwargs):
|
|
raise ServiceUnavailable('PRIVATE_DATABASE_ERROR')
|
|
monkeypatch.setattr(routes, 'search_graph', unavailable)
|
|
with TestClient(main.app) as client:
|
|
response = client.post('/graphrag', json={'query': 'Práca?'}, headers={'X-API-Key': KEY})
|
|
assert response.status_code == 503 and response.json()['reason'] == 'retrieval_unavailable'
|
|
assert 'PRIVATE_DATABASE_ERROR' not in response.text
|
|
response = client.post('/graphrag', json={'query': ' ', 'limit': 0}, headers={'X-API-Key': KEY})
|
|
assert response.status_code == 422 and response.json()['status'] == 'error'
|
|
assert client.post('/graphrag', json={'query': 'Práca?'}).status_code == 401
|