"""Standalone installation and real loopback HTTP transport, without an LLM.""" import asyncio import copy import hashlib import importlib.util import json from pathlib import Path import subprocess import sys import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import zipfile import pytest from fastapi.testclient import TestClient from neo4j.exceptions import ServiceUnavailable import app.main as main import app.routes as routes from scripts.build_tool_packages import build_package KEY = 'a' * 64 ROOT = Path(__file__).resolve().parents[1] def load_tool(engine, path=None): path = path or ROOT / 'packages/openwebui' / engine / 'tool.py' spec = importlib.util.spec_from_file_location('standalone_' + engine, path) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module @pytest.fixture(params=['rag', 'graphrag']) def engine(request): return request.param @pytest.fixture def evidence(): return {'status': 'ok', 'reason': None, 'complete': False, 'truncated': True, 'truncation_reasons': ['result_limit'], 'index_version': {'version': 'fixture'}, 'sources': [{'source_id': 'S1', 'document_path': 'pages/profile/README.md', 'source_url': 'https://example.test/profile', 'title': 'Eva Nová'}], 'context': 'Doložený text práce.', 'facts': [{'person': 'Eva Nová', 'year': 2024}], 'paths': [{'nodes': ['Eva Nová', 'Práca'], 'relationships': ['AUTHORED']}], 'aggregation': {'count': 1, 'complete': True}, 'constraints': {'years': [2024]}} @pytest.fixture def backend(monkeypatch, evidence): monkeypatch.setattr(main, 'embed_query', lambda _: None) calls = [] def retrieved(*args, **kwargs): calls.append((args, kwargs)) return copy.deepcopy(evidence) def rag_retrieved(*args, **kwargs): calls.append((args, kwargs)) result = {key: copy.deepcopy(value) for key, value in evidence.items() if key in routes.RagResponse.model_fields} return {**result, 'query': args[1], 'engine': 'hybrid_fts5_embeddings', 'source_count': len(evidence['sources']), 'strategies': ['fixture'], 'instructions': [], 'answer_format': {}} monkeypatch.setattr(routes, 'build_rag_context', rag_retrieved) monkeypatch.setattr(routes, 'compact_rag_payload', lambda result, **kwargs: result) monkeypatch.setattr(routes, 'search_graph', retrieved) state = {'requests': [], 'mode': 'api', 'code': 200, 'raw': b''} with TestClient(main.app) as client: class Handler(BaseHTTPRequestHandler): def log_message(self, *args): pass def do_POST(self): body = self.rfile.read(int(self.headers.get('Content-Length', '0'))) state['requests'].append({'path': self.path, 'body': json.loads(body), 'authorization': self.headers.get('Authorization')}) if state['mode'] == 'timeout': time.sleep(1.25) if state['mode'] == 'disconnect': self.close_connection = True return if state['mode'] == 'api': response = client.post(self.path.removeprefix('/prefix'), content=body, headers=dict(self.headers)) code, data = response.status_code, response.content else: code, data = state['code'], state['raw'] self.send_response(code) self.send_header('Content-Type', 'application/json') if code == 302: self.send_header('Location', state['base'] + '/redirected') self.end_headers() try: self.wfile.write(data) except OSError: pass server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) monkeypatch.setenv('ZP_TEST_HTTP_PORT', str(server.server_port)) state['base'] = f'http://127.0.0.1:{server.server_port}' thread = threading.Thread(target=server.serve_forever, kwargs={'poll_interval': 0.01}, daemon=True) thread.start() try: yield state, calls finally: server.shutdown() server.server_close() thread.join(timeout=3) def configured(engine, backend, path=None): module = load_tool(engine, path) tool = module.Tools() tool.valves.BACKEND_URL = backend[0]['base'] + '/prefix/' tool.valves.BACKEND_API_KEY = KEY return module, tool def invoke(tool, query='Akú prácu má Eva Nová v roku 2024?'): methods = [getattr(tool, name) for name in dir(tool) if name.startswith('retrieve_zpwiki_')] assert len(methods) == 1 return json.loads(asyncio.run(methods[0](query))) def test_actual_http_auth_query_and_all_evidence_survive(engine, backend, evidence): _, tool = configured(engine, backend) query = 'Akú prácu má Eva Nová v roku 2024?' result = invoke(tool, query) request = backend[0]['requests'][0] assert request['path'] == '/prefix/' + engine assert request['authorization'] == 'Bearer ' + KEY assert request['body']['query'] == query and request['body']['published_only'] is True assert result['status'] == 'ok' for key in ['sources', 'index_version', 'complete', 'truncated', 'truncation_reasons']: assert result[key] == evidence[key] assert result['requires_final_answer_verification'] is True if engine == 'rag': assert result['context'] == evidence['context'] assert request['body']['include_diagnostics'] is False else: assert result['facts'] == evidence['facts'] and result['paths'] == evidence['paths'] assert result['aggregation'] == evidence['aggregation'] assert backend[1][0][1] == {'limit': 40, 'max_entities': 8, 'published_only': True} def test_configuration_controls_scope_without_mutating_question(engine, backend): _, tool = configured(engine, backend) tool.valves.LIMIT = 2 tool.valves.PUBLISHED_ONLY = False invoke(tool, ' Osoba, rok a tag ') body = backend[0]['requests'][0]['body'] assert body['limit'] == 2 and body['published_only'] is False assert body['query'] == 'Osoba, rok a tag' @pytest.mark.parametrize('status', ['no_answer', 'ambiguous']) def test_no_answer_and_ambiguity_are_not_relabelled_as_positive(engine, backend, evidence, status): evidence.update(status=status, reason='no_matching_facts', sources=[], facts=[], paths=[], context='') _, tool = configured(engine, backend) result = invoke(tool) assert result['status'] == status and result['reason'] == 'no_matching_facts' assert result['sources'] == [] @pytest.mark.parametrize('query', ['', ' ', 'x' * 501, None]) def test_invalid_query_does_not_make_a_network_request(engine, backend, query): _, tool = configured(engine, backend) assert invoke(tool, query)['reason'] == 'invalid_query' assert backend[0]['requests'] == [] @pytest.mark.parametrize('url', ['', 'file:///tmp/file', 'https://user:password@example.test', 'http://example.test?query=1', 'http://example.test/#fragment']) def test_invalid_configuration_never_sends_credentials(engine, backend, url): _, tool = configured(engine, backend) tool.valves.BACKEND_URL = url assert invoke(tool)['reason'] == 'invalid_configuration' assert backend[0]['requests'] == [] @pytest.mark.parametrize('code,reason', [(401, 'unauthorized'), (403, 'forbidden'), (404, 'backend_endpoint_not_found'), (429, 'backend_busy'), (503, 'retrieval_unavailable'), (500, 'backend_http_error'), (302, 'redirect_not_allowed')]) def test_http_errors_do_not_expose_body_or_follow_redirect(engine, backend, code, reason): _, tool = configured(engine, backend) backend[0].update(mode='raw', code=code, raw=(KEY + ' PRIVATE_ERROR').encode()) result = invoke(tool) assert result['status'] == 'error' and result['reason'] == reason assert result['http_status'] == code and result['sources'] == [] assert KEY not in json.dumps(result) and 'PRIVATE_ERROR' not in json.dumps(result) assert len(backend[0]['requests']) == 1 def test_real_unauthorized_backend_rejects_wrong_key(engine, backend): _, tool = configured(engine, backend) tool.valves.BACKEND_API_KEY = 'wrong' assert invoke(tool)['reason'] == 'unauthorized' assert backend[1] == [] @pytest.mark.parametrize('raw', [b'not JSON', b'[]', b'{"status":"ok"}']) def test_invalid_backend_data_fails_closed(engine, backend, raw): _, tool = configured(engine, backend) backend[0].update(mode='raw', raw=raw) assert invoke(tool)['reason'] == 'invalid_backend_response' def test_contradictory_completeness_is_rejected(engine, backend, evidence): _, tool = configured(engine, backend) evidence.update(complete=True, truncated=True) assert invoke(tool)['reason'] == 'invalid_backend_response' def test_diagnostics_and_unverified_answer_are_not_passed_to_model(engine, backend, evidence): _, tool = configured(engine, backend) evidence.update(diagnostics={'private': 'PRIVATE'}, answer='UNVERIFIED', accepted_for_delivery=True) backend[0].update(mode='raw', raw=json.dumps(evidence).encode()) result = invoke(tool) assert 'diagnostics' not in result and 'answer' not in result and 'accepted_for_delivery' not in result def test_large_responses_are_rejected_without_partial_evidence(engine, backend, monkeypatch): module, tool = configured(engine, backend) monkeypatch.setattr(module, 'MAX_RESPONSE_BYTES', 100) backend[0].update(mode='raw', raw=b'x' * 101) result = invoke(tool) assert result['reason'] == 'response_too_large' and result['sources'] == [] def test_timeout_is_reported_without_raw_exception(engine, backend): _, tool = configured(engine, backend) tool.valves.TIMEOUT_SECONDS = 1 backend[0].update(mode='timeout') assert invoke(tool)['reason'] == 'backend_timeout' def test_connection_closed_before_response_returns_safe_error(engine, backend): _, tool = configured(engine, backend) backend[0]['mode'] = 'disconnect' assert invoke(tool)['reason'] == 'backend_unavailable' def test_graph_serialization_errors_have_shared_contract(backend, monkeypatch): monkeypatch.setattr(routes, 'search_graph', lambda *args, **kwargs: {'invalid': object()}) with TestClient(main.app) as client: response = client.post('/graphrag', json={'query': 'Práca?'}, headers={'X-API-Key': KEY}) assert response.status_code == 500 and response.json()['reason'] == 'internal_error' def test_graph_api_uses_shared_error_contract_and_validates_query(backend, monkeypatch): def unavailable(*args, **kwargs): raise ServiceUnavailable('PRIVATE_DATABASE_ERROR') monkeypatch.setattr(routes, 'search_graph', unavailable) with TestClient(main.app) as client: response = client.post('/graphrag', json={'query': 'Práca?'}, headers={'X-API-Key': KEY}) assert response.status_code == 503 and response.json()['reason'] == 'retrieval_unavailable' assert 'PRIVATE_DATABASE_ERROR' not in response.text response = client.post('/graphrag', json={'query': ' ', 'limit': 0}, headers={'X-API-Key': KEY}) assert response.status_code == 422 and response.json()['status'] == 'error' assert client.post('/graphrag', json={'query': 'Práca?'}).status_code == 401 def test_zip_and_json_installation_are_standalone_and_reproducible(engine, backend, tmp_path): built = build_package(engine, tmp_path / 'first') again = build_package(engine, tmp_path / 'again') assert built['sha256'] == again['sha256'] with zipfile.ZipFile(built['archive']) as archive: manifest = json.loads(archive.read('manifest.json')) for name, digest in manifest['files'].items(): assert hashlib.sha256(archive.read(name)).hexdigest() == digest form = json.loads(archive.read('openwebui-tool.json')) assert len(form) == 1 and form[0]['id'] == 'zpwiki_' + engine assert form[0]['access_grants'] == [] assert form[0]['content'].encode() == archive.read('tool.py') assert Path(built['import_file']).read_bytes() == archive.read('openwebui-tool.json') extracted = tmp_path / 'isolated.py' extracted.write_bytes(archive.read('tool.py')) code = "import pathlib; source=pathlib.Path(__import__('sys').argv[1]).read_text(); scope={'__name__':'standalone_tool'}; exec(source,scope); tool=scope['Tools'](); assert len([n for n in dir(tool) if n.startswith('retrieve_zpwiki_')])==1" result = subprocess.run([sys.executable, '-I', '-c', code, str(extracted)], cwd=tmp_path, capture_output=True, text=True, timeout=20) assert result.returncode == 0, result.stderr _, tool = configured(engine, backend, extracted) assert invoke(tool)['status'] == 'ok'