284 lines
13 KiB
Python
284 lines
13 KiB
Python
"""Standalone installation and real loopback HTTP transport, without an LLM."""
|
|
import asyncio
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import zipfile
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from neo4j.exceptions import ServiceUnavailable
|
|
|
|
import app.main as main
|
|
import app.routes as routes
|
|
from scripts.build_tool_packages import build_package
|
|
|
|
KEY = 'a' * 64
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def load_tool(engine, path=None):
|
|
path = path or ROOT / 'packages/openwebui' / engine / 'tool.py'
|
|
spec = importlib.util.spec_from_file_location('standalone_' + engine, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(params=['rag', 'graphrag'])
|
|
def engine(request):
|
|
return request.param
|
|
|
|
|
|
@pytest.fixture
|
|
def evidence():
|
|
return {'status': 'ok', 'reason': None, 'complete': False, 'truncated': True,
|
|
'truncation_reasons': ['result_limit'], 'index_version': {'version': 'fixture'},
|
|
'sources': [{'source_id': 'S1', 'document_path': 'pages/profile/README.md',
|
|
'source_url': 'https://example.test/profile', 'title': 'Eva Nová'}],
|
|
'context': 'Doložený text práce.', 'facts': [{'person': 'Eva Nová', 'year': 2024}],
|
|
'paths': [{'nodes': ['Eva Nová', 'Práca'], 'relationships': ['AUTHORED']}],
|
|
'aggregation': {'count': 1, 'complete': True}, 'constraints': {'years': [2024]}}
|
|
|
|
|
|
@pytest.fixture
|
|
def backend(monkeypatch, evidence):
|
|
monkeypatch.setattr(main, 'embed_query', lambda _: None)
|
|
calls = []
|
|
def retrieved(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return copy.deepcopy(evidence)
|
|
def rag_retrieved(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
result = {key: copy.deepcopy(value) for key, value in evidence.items()
|
|
if key in routes.RagResponse.model_fields}
|
|
return {**result, 'query': args[1], 'engine': 'hybrid_fts5_embeddings',
|
|
'source_count': len(evidence['sources']), 'strategies': ['fixture'],
|
|
'instructions': [], 'answer_format': {}}
|
|
monkeypatch.setattr(routes, 'build_rag_context', rag_retrieved)
|
|
monkeypatch.setattr(routes, 'compact_rag_payload', lambda result, **kwargs: result)
|
|
monkeypatch.setattr(routes, 'search_graph', retrieved)
|
|
state = {'requests': [], 'mode': 'api', 'code': 200, 'raw': b''}
|
|
with TestClient(main.app) as client:
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *args):
|
|
pass
|
|
def do_POST(self):
|
|
body = self.rfile.read(int(self.headers.get('Content-Length', '0')))
|
|
state['requests'].append({'path': self.path, 'body': json.loads(body),
|
|
'authorization': self.headers.get('Authorization')})
|
|
if state['mode'] == 'timeout':
|
|
time.sleep(1.25)
|
|
if state['mode'] == 'disconnect':
|
|
self.close_connection = True
|
|
return
|
|
if state['mode'] == 'api':
|
|
response = client.post(self.path.removeprefix('/prefix'), content=body,
|
|
headers=dict(self.headers))
|
|
code, data = response.status_code, response.content
|
|
else:
|
|
code, data = state['code'], state['raw']
|
|
self.send_response(code)
|
|
self.send_header('Content-Type', 'application/json')
|
|
if code == 302:
|
|
self.send_header('Location', state['base'] + '/redirected')
|
|
self.end_headers()
|
|
try:
|
|
self.wfile.write(data)
|
|
except OSError:
|
|
pass
|
|
server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
|
monkeypatch.setenv('ZP_TEST_HTTP_PORT', str(server.server_port))
|
|
state['base'] = f'http://127.0.0.1:{server.server_port}'
|
|
thread = threading.Thread(target=server.serve_forever, kwargs={'poll_interval': 0.01}, daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield state, calls
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join(timeout=3)
|
|
|
|
|
|
def configured(engine, backend, path=None):
|
|
module = load_tool(engine, path)
|
|
tool = module.Tools()
|
|
tool.valves.BACKEND_URL = backend[0]['base'] + '/prefix/'
|
|
tool.valves.BACKEND_API_KEY = KEY
|
|
return module, tool
|
|
|
|
|
|
def invoke(tool, query='Akú prácu má Eva Nová v roku 2024?'):
|
|
methods = [getattr(tool, name) for name in dir(tool) if name.startswith('retrieve_zpwiki_')]
|
|
assert len(methods) == 1
|
|
return json.loads(asyncio.run(methods[0](query)))
|
|
|
|
|
|
def test_actual_http_auth_query_and_all_evidence_survive(engine, backend, evidence):
|
|
_, tool = configured(engine, backend)
|
|
query = 'Akú prácu má Eva Nová v roku 2024?'
|
|
result = invoke(tool, query)
|
|
request = backend[0]['requests'][0]
|
|
assert request['path'] == '/prefix/' + engine
|
|
assert request['authorization'] == 'Bearer ' + KEY
|
|
assert request['body']['query'] == query and request['body']['published_only'] is True
|
|
assert result['status'] == 'ok'
|
|
for key in ['sources', 'index_version', 'complete', 'truncated', 'truncation_reasons']:
|
|
assert result[key] == evidence[key]
|
|
assert result['requires_final_answer_verification'] is True
|
|
if engine == 'rag':
|
|
assert result['context'] == evidence['context']
|
|
assert request['body']['include_diagnostics'] is False
|
|
else:
|
|
assert result['facts'] == evidence['facts'] and result['paths'] == evidence['paths']
|
|
assert result['aggregation'] == evidence['aggregation']
|
|
assert backend[1][0][1] == {'limit': 40, 'max_entities': 8, 'published_only': True}
|
|
|
|
|
|
def test_configuration_controls_scope_without_mutating_question(engine, backend):
|
|
_, tool = configured(engine, backend)
|
|
tool.valves.LIMIT = 2
|
|
tool.valves.PUBLISHED_ONLY = False
|
|
invoke(tool, ' Osoba, rok a tag ')
|
|
body = backend[0]['requests'][0]['body']
|
|
assert body['limit'] == 2 and body['published_only'] is False
|
|
assert body['query'] == 'Osoba, rok a tag'
|
|
|
|
|
|
@pytest.mark.parametrize('status', ['no_answer', 'ambiguous'])
|
|
def test_no_answer_and_ambiguity_are_not_relabelled_as_positive(engine, backend, evidence, status):
|
|
evidence.update(status=status, reason='no_matching_facts', sources=[], facts=[], paths=[], context='')
|
|
_, tool = configured(engine, backend)
|
|
result = invoke(tool)
|
|
assert result['status'] == status and result['reason'] == 'no_matching_facts'
|
|
assert result['sources'] == []
|
|
|
|
|
|
@pytest.mark.parametrize('query', ['', ' ', 'x' * 501, None])
|
|
def test_invalid_query_does_not_make_a_network_request(engine, backend, query):
|
|
_, tool = configured(engine, backend)
|
|
assert invoke(tool, query)['reason'] == 'invalid_query'
|
|
assert backend[0]['requests'] == []
|
|
|
|
|
|
@pytest.mark.parametrize('url', ['', 'file:///tmp/file', 'https://user:password@example.test',
|
|
'http://example.test?query=1', 'http://example.test/#fragment'])
|
|
def test_invalid_configuration_never_sends_credentials(engine, backend, url):
|
|
_, tool = configured(engine, backend)
|
|
tool.valves.BACKEND_URL = url
|
|
assert invoke(tool)['reason'] == 'invalid_configuration'
|
|
assert backend[0]['requests'] == []
|
|
|
|
|
|
@pytest.mark.parametrize('code,reason', [(401, 'unauthorized'), (403, 'forbidden'),
|
|
(404, 'backend_endpoint_not_found'), (429, 'backend_busy'), (503, 'retrieval_unavailable'),
|
|
(500, 'backend_http_error'), (302, 'redirect_not_allowed')])
|
|
def test_http_errors_do_not_expose_body_or_follow_redirect(engine, backend, code, reason):
|
|
_, tool = configured(engine, backend)
|
|
backend[0].update(mode='raw', code=code, raw=(KEY + ' PRIVATE_ERROR').encode())
|
|
result = invoke(tool)
|
|
assert result['status'] == 'error' and result['reason'] == reason
|
|
assert result['http_status'] == code and result['sources'] == []
|
|
assert KEY not in json.dumps(result) and 'PRIVATE_ERROR' not in json.dumps(result)
|
|
assert len(backend[0]['requests']) == 1
|
|
|
|
|
|
def test_real_unauthorized_backend_rejects_wrong_key(engine, backend):
|
|
_, tool = configured(engine, backend)
|
|
tool.valves.BACKEND_API_KEY = 'wrong'
|
|
assert invoke(tool)['reason'] == 'unauthorized'
|
|
assert backend[1] == []
|
|
|
|
|
|
@pytest.mark.parametrize('raw', [b'<html>not JSON</html>', b'[]', b'{"status":"ok"}'])
|
|
def test_invalid_backend_data_fails_closed(engine, backend, raw):
|
|
_, tool = configured(engine, backend)
|
|
backend[0].update(mode='raw', raw=raw)
|
|
assert invoke(tool)['reason'] == 'invalid_backend_response'
|
|
|
|
|
|
def test_contradictory_completeness_is_rejected(engine, backend, evidence):
|
|
_, tool = configured(engine, backend)
|
|
evidence.update(complete=True, truncated=True)
|
|
assert invoke(tool)['reason'] == 'invalid_backend_response'
|
|
|
|
|
|
def test_diagnostics_and_unverified_answer_are_not_passed_to_model(engine, backend, evidence):
|
|
_, tool = configured(engine, backend)
|
|
evidence.update(diagnostics={'private': 'PRIVATE'}, answer='UNVERIFIED', accepted_for_delivery=True)
|
|
backend[0].update(mode='raw', raw=json.dumps(evidence).encode())
|
|
result = invoke(tool)
|
|
assert 'diagnostics' not in result and 'answer' not in result and 'accepted_for_delivery' not in result
|
|
|
|
|
|
def test_large_responses_are_rejected_without_partial_evidence(engine, backend, monkeypatch):
|
|
module, tool = configured(engine, backend)
|
|
monkeypatch.setattr(module, 'MAX_RESPONSE_BYTES', 100)
|
|
backend[0].update(mode='raw', raw=b'x' * 101)
|
|
result = invoke(tool)
|
|
assert result['reason'] == 'response_too_large' and result['sources'] == []
|
|
|
|
|
|
def test_timeout_is_reported_without_raw_exception(engine, backend):
|
|
_, tool = configured(engine, backend)
|
|
tool.valves.TIMEOUT_SECONDS = 1
|
|
backend[0].update(mode='timeout')
|
|
assert invoke(tool)['reason'] == 'backend_timeout'
|
|
|
|
|
|
def test_connection_closed_before_response_returns_safe_error(engine, backend):
|
|
_, tool = configured(engine, backend)
|
|
backend[0]['mode'] = 'disconnect'
|
|
assert invoke(tool)['reason'] == 'backend_unavailable'
|
|
|
|
|
|
def test_graph_serialization_errors_have_shared_contract(backend, monkeypatch):
|
|
monkeypatch.setattr(routes, 'search_graph', lambda *args, **kwargs: {'invalid': object()})
|
|
with TestClient(main.app) as client:
|
|
response = client.post('/graphrag', json={'query': 'Práca?'}, headers={'X-API-Key': KEY})
|
|
assert response.status_code == 500 and response.json()['reason'] == 'internal_error'
|
|
|
|
|
|
def test_graph_api_uses_shared_error_contract_and_validates_query(backend, monkeypatch):
|
|
def unavailable(*args, **kwargs):
|
|
raise ServiceUnavailable('PRIVATE_DATABASE_ERROR')
|
|
monkeypatch.setattr(routes, 'search_graph', unavailable)
|
|
with TestClient(main.app) as client:
|
|
response = client.post('/graphrag', json={'query': 'Práca?'}, headers={'X-API-Key': KEY})
|
|
assert response.status_code == 503 and response.json()['reason'] == 'retrieval_unavailable'
|
|
assert 'PRIVATE_DATABASE_ERROR' not in response.text
|
|
response = client.post('/graphrag', json={'query': ' ', 'limit': 0}, headers={'X-API-Key': KEY})
|
|
assert response.status_code == 422 and response.json()['status'] == 'error'
|
|
assert client.post('/graphrag', json={'query': 'Práca?'}).status_code == 401
|
|
|
|
|
|
def test_zip_and_json_installation_are_standalone_and_reproducible(engine, backend, tmp_path):
|
|
built = build_package(engine, tmp_path / 'first')
|
|
again = build_package(engine, tmp_path / 'again')
|
|
assert built['sha256'] == again['sha256']
|
|
with zipfile.ZipFile(built['archive']) as archive:
|
|
manifest = json.loads(archive.read('manifest.json'))
|
|
for name, digest in manifest['files'].items():
|
|
assert hashlib.sha256(archive.read(name)).hexdigest() == digest
|
|
form = json.loads(archive.read('openwebui-tool.json'))
|
|
assert len(form) == 1 and form[0]['id'] == 'zpwiki_' + engine
|
|
assert form[0]['access_grants'] == []
|
|
assert form[0]['content'].encode() == archive.read('tool.py')
|
|
assert Path(built['import_file']).read_bytes() == archive.read('openwebui-tool.json')
|
|
extracted = tmp_path / 'isolated.py'
|
|
extracted.write_bytes(archive.read('tool.py'))
|
|
code = "import pathlib; source=pathlib.Path(__import__('sys').argv[1]).read_text(); scope={'__name__':'standalone_tool'}; exec(source,scope); tool=scope['Tools'](); assert len([n for n in dir(tool) if n.startswith('retrieve_zpwiki_')])==1"
|
|
result = subprocess.run([sys.executable, '-I', '-c', code, str(extracted)],
|
|
cwd=tmp_path, capture_output=True, text=True, timeout=20)
|
|
assert result.returncode == 0, result.stderr
|
|
_, tool = configured(engine, backend, extracted)
|
|
assert invoke(tool)['status'] == 'ok'
|