242 lines
11 KiB
Python
242 lines
11 KiB
Python
"""Failure-injection tests for coordinated publication, not only happy paths."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.sync import index_state as state
|
|
from tools.sync import sync_indexes as sync
|
|
|
|
|
|
class Result:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
def single(self):
|
|
return self.value
|
|
|
|
|
|
class Session:
|
|
def __init__(self, version):
|
|
self.version = version; self.fail_commit = None; self.unavailable = False
|
|
def run(self, query, **params):
|
|
if self.unavailable:
|
|
raise ConnectionError('lost database connection')
|
|
return Result({'metadata': json.dumps(self.version)} if self.version else None)
|
|
def begin_transaction(self):
|
|
return Transaction(self)
|
|
|
|
|
|
class Transaction:
|
|
def __init__(self, session):
|
|
self.session = session; self.version = session.version
|
|
def __enter__(self):
|
|
return self
|
|
def __exit__(self, *args):
|
|
pass # Uncommitted local state is discarded, as in the Neo4j driver.
|
|
def commit(self):
|
|
if self.session.fail_commit == 'unknown':
|
|
self.session.unavailable = True
|
|
raise ConnectionError('commit outcome unknown')
|
|
self.session.version = self.version
|
|
if self.session.fail_commit == 'response_lost':
|
|
raise ConnectionError('committed but response lost')
|
|
|
|
|
|
def snapshot(directory: Path, version: dict):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / 'documents.json').write_text(json.dumps([version]))
|
|
(directory / 'chunks.json').write_text(json.dumps([version]))
|
|
with sqlite3.connect(directory / 'zp_index.sqlite') as conn:
|
|
conn.execute('CREATE TABLE index_state (id INTEGER PRIMARY KEY, metadata_json TEXT)')
|
|
conn.execute('INSERT INTO index_state VALUES (1,?)', (json.dumps(version),))
|
|
state.durable_json(directory / state.VERSION_FILE, version)
|
|
|
|
|
|
@pytest.fixture
|
|
def publication(tmp_path, monkeypatch):
|
|
data = tmp_path / 'data'; stage = data / '.sync-stage-test'
|
|
old = {'version': 'old', 'git_revision': 'commit-old'}
|
|
new = {'version': 'new', 'git_revision': 'commit-new'}
|
|
snapshot(data, old); snapshot(stage, new)
|
|
monkeypatch.setattr(sync, 'write_graph_transaction', lambda tx, payload, version: setattr(tx, 'version', version))
|
|
monkeypatch.setattr(sync, 'graph_counts', lambda tx: ([('Person', 1)], [('HAS_DOCUMENT', 1)]))
|
|
return data, stage, old, new, Session(old)
|
|
|
|
|
|
def test_publish_makes_all_versions_agree(publication):
|
|
data, stage, old, new, session = publication
|
|
result = sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert result == {'nodes': 1, 'relationships': 1}
|
|
assert state.read_version(data) == state.sqlite_version(data / 'zp_index.sqlite') == session.version == new
|
|
with state.index_snapshot(data, db_file=data / 'zp_index.sqlite') as version:
|
|
assert version == new
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_graph_write_failure_keeps_original_files_and_graph(publication, monkeypatch):
|
|
data, stage, old, new, session = publication
|
|
before = {name: (data / name).read_bytes() for name in sync.FILES}
|
|
def fail(tx, payload, version):
|
|
tx.version = new
|
|
raise RuntimeError('failure halfway through graph write')
|
|
monkeypatch.setattr(sync, 'write_graph_transaction', fail)
|
|
with pytest.raises(RuntimeError, match='halfway'):
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert session.version == old
|
|
assert before == {name: (data / name).read_bytes() for name in sync.FILES}
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_partial_file_publication_rolls_back_files_and_graph(publication, monkeypatch):
|
|
data, stage, old, new, session = publication
|
|
before = {name: (data / name).read_bytes() for name in sync.FILES}
|
|
replace = sync.replace_file
|
|
def fail(source, target):
|
|
if target.name == 'zp_index.sqlite':
|
|
raise OSError('disk write failed')
|
|
replace(source, target)
|
|
monkeypatch.setattr(sync, 'replace_file', fail)
|
|
with pytest.raises(OSError, match='disk write'):
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert session.version == old
|
|
assert before == {name: (data / name).read_bytes() for name in sync.FILES}
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_lost_commit_response_is_verified_before_restoring_any_files(publication):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = 'response_lost'
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert session.version == state.sqlite_version(data / 'zp_index.sqlite') == new
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_unknown_commit_blocks_reads_until_next_successful_sync(publication):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = 'unknown'
|
|
with pytest.raises(ConnectionError):
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert (stage / 'previous/zp_index.sqlite').exists()
|
|
with pytest.raises(state.IndexUnavailableError, match='nebola dokončená'):
|
|
with state.index_snapshot(data):
|
|
pytest.fail('Mixed generation must never be readable')
|
|
session.unavailable = False; session.fail_commit = None
|
|
repaired = {'version': 'repaired'}; next_stage = data / '.sync-stage-repaired'
|
|
snapshot(next_stage, repaired)
|
|
sync.publish_snapshot(data, next_stage, repaired, {}, session)
|
|
assert session.version == state.sqlite_version(data / 'zp_index.sqlite') == state.read_version(data) == repaired
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_read_during_publication_is_unavailable(publication):
|
|
data, *_ = publication
|
|
with state.publication_lock(data, exclusive=True):
|
|
with pytest.raises(state.IndexUnavailableError, match='aktivácia'):
|
|
with state.index_snapshot(data):
|
|
pytest.fail('Publication barrier bypassed')
|
|
|
|
|
|
def test_reader_rejects_a_replaced_sqlite_database(publication):
|
|
data, stage, old, new, session = publication
|
|
state.durable_json(data / state.VERSION_FILE, new)
|
|
with pytest.raises(state.IndexUnavailableError, match='SQLite'):
|
|
with state.index_snapshot(data, db_file=data / 'zp_index.sqlite'):
|
|
pass
|
|
|
|
|
|
def test_staging_failure_never_touches_live_indices(tmp_path, monkeypatch):
|
|
data = tmp_path / 'data'; snapshot(data, {'version': 'old'})
|
|
before = {name: (data / name).read_bytes() for name in sync.FILES}
|
|
def fail(wiki, stage):
|
|
(stage / 'documents.json').write_text('new incomplete data')
|
|
raise RuntimeError('embedding failed')
|
|
monkeypatch.setattr(sync, 'prepare_snapshot', fail)
|
|
with pytest.raises(RuntimeError, match='embedding failed'):
|
|
sync.synchronize_indexes(tmp_path / 'wiki', data)
|
|
assert before == {name: (data / name).read_bytes() for name in sync.FILES}
|
|
assert not list(data.glob('.sync-stage-*'))
|
|
|
|
|
|
def test_prepare_uses_one_frozen_source_and_records_versions(tmp_path, monkeypatch):
|
|
import numpy as np
|
|
import tools.rag.build_sqlite_index as indexer
|
|
wiki = tmp_path / 'wiki'; page = wiki / 'pages/students/2020/eva/README.md'
|
|
page.parent.mkdir(parents=True)
|
|
page.write_text('---\ntitle: Eva Nová\npublished: true\ntaxonomy:\n category: [bp2024]\n---\n# Bakalárska práca 2024\n\nNázov: Overovanie synchronizácie indexov\n')
|
|
monkeypatch.setattr(indexer, 'embed_passages', lambda texts: np.ones((len(texts), 4), dtype=np.float32))
|
|
stage = tmp_path / 'stage'; stage.mkdir()
|
|
version, counts, payload = sync.prepare_snapshot(wiki, stage)
|
|
assert counts['documents'] == 1 and counts['chunks'] > 0
|
|
assert version == state.sqlite_version(stage / 'zp_index.sqlite') == state.read_version(stage)
|
|
assert version['documents_sha256'] == sync.digest(stage / 'documents.json')
|
|
assert version['chunks_sha256'] == sync.digest(stage / 'chunks.json')
|
|
assert payload['people'][0]['name'] == 'Eva Nová'
|
|
assert payload['theses'][0]['year'] == 2024
|
|
assert payload['documents'][0]['path'] == 'pages/students/2020/eva/README.md'
|
|
|
|
|
|
def test_changed_source_is_not_activated(tmp_path, monkeypatch):
|
|
import numpy as np
|
|
import tools.rag.build_sqlite_index as indexer
|
|
wiki = tmp_path / 'wiki'; page = wiki / 'pages/example/README.md'
|
|
page.parent.mkdir(parents=True); page.write_text('---\ntitle: Source\n---\nInitial source text.')
|
|
def embed(texts):
|
|
page.write_text('Source changed during indexing.')
|
|
return np.ones((len(texts), 4), dtype=np.float32)
|
|
monkeypatch.setattr(indexer, 'embed_passages', embed)
|
|
stage = tmp_path / 'stage'; stage.mkdir()
|
|
with pytest.raises(RuntimeError, match='počas indexovania'):
|
|
sync.prepare_snapshot(wiki, stage)
|
|
|
|
|
|
def test_api_returns_503_while_publication_is_unresolved(tmp_path, monkeypatch):
|
|
from fastapi.testclient import TestClient
|
|
import app.main as main
|
|
import app.routes as routes
|
|
monkeypatch.setattr(main, 'embed_query', lambda q: None)
|
|
for name, value in [('WEBHOOK_SECRET', 'w'), ('SEARCH_API_KEY', 'a'), ('SYNC_API_KEY', 's')]:
|
|
monkeypatch.setenv(name, value * 64)
|
|
data = tmp_path / 'data'; snapshot(data, {'version': 'old'})
|
|
state.durable_json(data / state.PENDING_FILE, {'target_version': 'new'})
|
|
monkeypatch.setattr(routes, 'DB_FILE', data / 'zp_index.sqlite')
|
|
with TestClient(main.app) as client:
|
|
response = client.post('/rag', json={'query': 'Aká práca?'},
|
|
headers={'Authorization': 'Bearer ' + 'a' * 64})
|
|
assert response.status_code == 503
|
|
assert client.get('/health').json()['ready'] is False
|
|
|
|
|
|
@pytest.mark.parametrize('endpoint', ['sync', 'webhook/gitea'])
|
|
def test_sync_api_exposes_common_version_and_graph_counts(monkeypatch, endpoint):
|
|
from fastapi.testclient import TestClient
|
|
import app.main as main
|
|
import app.routes as routes
|
|
monkeypatch.setattr(main, 'embed_query', lambda q: None)
|
|
for name, value in [('WEBHOOK_SECRET', 'w'), ('SEARCH_API_KEY', 'a'), ('SYNC_API_KEY', 's')]:
|
|
monkeypatch.setenv(name, value * 64)
|
|
monkeypatch.setenv('EXPECTED_GITEA_REPOSITORY', 'KEMT/zpwiki')
|
|
monkeypatch.setenv('WEBHOOK_PULL_GIT', 'false')
|
|
version = {'version': 'new', 'git_revision': 'commit'}
|
|
monkeypatch.setattr(main, 'rebuild_index', lambda **kwargs: {
|
|
'duration_seconds': 0.1, 'counts': {'documents': 1}, 'index_version': version,
|
|
'graph_counts': {'nodes': 2, 'relationships': 1}})
|
|
with TestClient(main.app) as client:
|
|
if endpoint == 'sync':
|
|
response = client.post('/sync', json={}, headers={'X-API-Key': 's' * 64})
|
|
else:
|
|
body = json.dumps({'repository': {'full_name': 'KEMT/zpwiki'}, 'ref': 'refs/heads/master'}).encode()
|
|
signature = hmac.new(('w' * 64).encode(), body, hashlib.sha256).hexdigest()
|
|
response = client.post('/webhook/gitea', content=body,
|
|
headers={'X-Gitea-Event': 'push', 'X-Gitea-Signature': signature, 'Content-Type': 'application/json'})
|
|
assert response.status_code == 202
|
|
main.app.state.sync_jobs.run_once()
|
|
job = client.get(response.json()['status_url'], headers={'X-API-Key': 's' * 64}).json()
|
|
assert job['status'] == 'succeeded'
|
|
assert job['result']['index_version'] == version
|
|
assert job['result']['graph_counts'] == {'nodes': 2, 'relationships': 1}
|