310 lines
12 KiB
Python
310 lines
12 KiB
Python
"""Failure-injection tests for coordinated publication, not only happy paths."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.sync import index_state as state
|
|
from tools.sync import sync_indexes as sync
|
|
|
|
|
|
class Result:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def single(self):
|
|
return self.value
|
|
|
|
|
|
class Session:
|
|
def __init__(self, version):
|
|
self.version = version
|
|
self.fail_commit = None
|
|
self.unavailable = False
|
|
|
|
def run(self, query, **params):
|
|
if self.unavailable:
|
|
raise ConnectionError("lost database connection")
|
|
return Result({"metadata": json.dumps(self.version)} if self.version else None)
|
|
|
|
def begin_transaction(self):
|
|
return Transaction(self)
|
|
|
|
|
|
class Transaction:
|
|
def __init__(self, session):
|
|
self.session = session
|
|
self.version = session.version
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
pass # Uncommitted local state is discarded, as in the Neo4j driver.
|
|
|
|
def commit(self):
|
|
if self.session.fail_commit == "unknown":
|
|
self.session.unavailable = True
|
|
raise ConnectionError("commit outcome unknown")
|
|
self.session.version = self.version
|
|
if self.session.fail_commit == "response_lost":
|
|
raise ConnectionError("committed but response lost")
|
|
|
|
|
|
def snapshot(directory: Path, version: dict):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / "documents.json").write_text(json.dumps([version]))
|
|
(directory / "chunks.json").write_text(json.dumps([version]))
|
|
with sqlite3.connect(directory / "zp_index.sqlite") as conn:
|
|
conn.execute("CREATE TABLE index_state (id INTEGER PRIMARY KEY, metadata_json TEXT)")
|
|
conn.execute("INSERT INTO index_state VALUES (1,?)", (json.dumps(version),))
|
|
state.durable_json(directory / state.VERSION_FILE, version)
|
|
|
|
|
|
@pytest.fixture
|
|
def publication(tmp_path, monkeypatch):
|
|
data = tmp_path / "data"
|
|
stage = data / ".sync-stage-test"
|
|
old = {"version": "old", "git_revision": "commit-old"}
|
|
new = {"version": "new", "git_revision": "commit-new"}
|
|
snapshot(data, old)
|
|
snapshot(stage, new)
|
|
monkeypatch.setattr(
|
|
sync,
|
|
"write_graph_transaction",
|
|
lambda tx, payload, version: setattr(tx, "version", version),
|
|
)
|
|
monkeypatch.setattr(sync, "graph_counts", lambda tx: ([("Person", 1)], [("HAS_DOCUMENT", 1)]))
|
|
return data, stage, old, new, Session(old)
|
|
|
|
|
|
def test_publish_makes_all_versions_agree(publication):
|
|
data, stage, old, new, session = publication
|
|
result = sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert result == {"nodes": 1, "relationships": 1}
|
|
assert (
|
|
state.read_version(data)
|
|
== state.sqlite_version(data / "zp_index.sqlite")
|
|
== session.version
|
|
== new
|
|
)
|
|
with state.index_snapshot(data, db_file=data / "zp_index.sqlite") as version:
|
|
assert version == new
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_graph_write_failure_keeps_original_files_and_graph(publication, monkeypatch):
|
|
data, stage, old, new, session = publication
|
|
before = {name: (data / name).read_bytes() for name in sync.FILES}
|
|
|
|
def fail(tx, payload, version):
|
|
tx.version = new
|
|
raise RuntimeError("failure halfway through graph write")
|
|
|
|
monkeypatch.setattr(sync, "write_graph_transaction", fail)
|
|
with pytest.raises(RuntimeError, match="halfway"):
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert session.version == old
|
|
assert before == {name: (data / name).read_bytes() for name in sync.FILES}
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_partial_file_publication_rolls_back_files_and_graph(publication, monkeypatch):
|
|
data, stage, old, new, session = publication
|
|
before = {name: (data / name).read_bytes() for name in sync.FILES}
|
|
replace = sync.replace_file
|
|
|
|
def fail(source, target):
|
|
if target.name == "zp_index.sqlite":
|
|
raise OSError("disk write failed")
|
|
replace(source, target)
|
|
|
|
monkeypatch.setattr(sync, "replace_file", fail)
|
|
with pytest.raises(OSError, match="disk write"):
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert session.version == old
|
|
assert before == {name: (data / name).read_bytes() for name in sync.FILES}
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_lost_commit_response_is_verified_before_restoring_any_files(publication):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = "response_lost"
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert session.version == state.sqlite_version(data / "zp_index.sqlite") == new
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_unknown_commit_blocks_reads_until_next_successful_sync(publication):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = "unknown"
|
|
with pytest.raises(ConnectionError):
|
|
sync.publish_snapshot(data, stage, new, {}, session)
|
|
assert (stage / "previous/zp_index.sqlite").exists()
|
|
with pytest.raises(state.IndexUnavailableError, match="nebola dokončená"):
|
|
with state.index_snapshot(data):
|
|
pytest.fail("Mixed generation must never be readable")
|
|
session.unavailable = False
|
|
session.fail_commit = None
|
|
repaired = {"version": "repaired"}
|
|
next_stage = data / ".sync-stage-repaired"
|
|
snapshot(next_stage, repaired)
|
|
sync.publish_snapshot(data, next_stage, repaired, {}, session)
|
|
assert (
|
|
session.version
|
|
== state.sqlite_version(data / "zp_index.sqlite")
|
|
== state.read_version(data)
|
|
== repaired
|
|
)
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_read_during_publication_is_unavailable(publication):
|
|
data, *_ = publication
|
|
with state.publication_lock(data, exclusive=True):
|
|
with pytest.raises(state.IndexUnavailableError, match="aktivácia"):
|
|
with state.index_snapshot(data):
|
|
pytest.fail("Publication barrier bypassed")
|
|
|
|
|
|
def test_reader_rejects_a_replaced_sqlite_database(publication):
|
|
data, stage, old, new, session = publication
|
|
state.durable_json(data / state.VERSION_FILE, new)
|
|
with pytest.raises(state.IndexUnavailableError, match="SQLite"):
|
|
with state.index_snapshot(data, db_file=data / "zp_index.sqlite"):
|
|
pass
|
|
|
|
|
|
def test_staging_failure_never_touches_live_indices(tmp_path, monkeypatch):
|
|
data = tmp_path / "data"
|
|
snapshot(data, {"version": "old"})
|
|
before = {name: (data / name).read_bytes() for name in sync.FILES}
|
|
|
|
def fail(wiki, stage):
|
|
(stage / "documents.json").write_text("new incomplete data")
|
|
raise RuntimeError("embedding failed")
|
|
|
|
monkeypatch.setattr(sync, "prepare_snapshot", fail)
|
|
with pytest.raises(RuntimeError, match="embedding failed"):
|
|
sync.synchronize_indexes(tmp_path / "wiki", data)
|
|
assert before == {name: (data / name).read_bytes() for name in sync.FILES}
|
|
assert not list(data.glob(".sync-stage-*"))
|
|
|
|
|
|
def test_prepare_uses_one_frozen_source_and_records_versions(tmp_path, monkeypatch):
|
|
import numpy as np
|
|
import tools.rag.build_sqlite_index as indexer
|
|
|
|
wiki = tmp_path / "wiki"
|
|
page = wiki / "pages/students/2020/eva/README.md"
|
|
page.parent.mkdir(parents=True)
|
|
page.write_text(
|
|
"---\ntitle: Eva Nová\npublished: true\ntaxonomy:\n category: [bp2024]\n---\n# Bakalárska práca 2024\n\nNázov: Overovanie synchronizácie indexov\n"
|
|
)
|
|
monkeypatch.setattr(
|
|
indexer, "embed_passages", lambda texts: np.ones((len(texts), 4), dtype=np.float32)
|
|
)
|
|
stage = tmp_path / "stage"
|
|
stage.mkdir()
|
|
version, counts, payload = sync.prepare_snapshot(wiki, stage)
|
|
assert counts["documents"] == 1 and counts["chunks"] > 0
|
|
assert version == state.sqlite_version(stage / "zp_index.sqlite") == state.read_version(stage)
|
|
assert version["documents_sha256"] == sync.digest(stage / "documents.json")
|
|
assert version["chunks_sha256"] == sync.digest(stage / "chunks.json")
|
|
assert payload["people"][0]["name"] == "Eva Nová"
|
|
assert payload["theses"][0]["year"] == 2024
|
|
assert payload["documents"][0]["path"] == "pages/students/2020/eva/README.md"
|
|
|
|
|
|
def test_changed_source_is_not_activated(tmp_path, monkeypatch):
|
|
import numpy as np
|
|
import tools.rag.build_sqlite_index as indexer
|
|
|
|
wiki = tmp_path / "wiki"
|
|
page = wiki / "pages/example/README.md"
|
|
page.parent.mkdir(parents=True)
|
|
page.write_text("---\ntitle: Source\n---\nInitial source text.")
|
|
|
|
def embed(texts):
|
|
page.write_text("Source changed during indexing.")
|
|
return np.ones((len(texts), 4), dtype=np.float32)
|
|
|
|
monkeypatch.setattr(indexer, "embed_passages", embed)
|
|
stage = tmp_path / "stage"
|
|
stage.mkdir()
|
|
with pytest.raises(RuntimeError, match="počas indexovania"):
|
|
sync.prepare_snapshot(wiki, stage)
|
|
|
|
|
|
def test_api_returns_503_while_publication_is_unresolved(tmp_path, monkeypatch):
|
|
from fastapi.testclient import TestClient
|
|
import app.main as main
|
|
import app.routes as routes
|
|
|
|
monkeypatch.setattr(main, "embed_query", lambda q: None)
|
|
for name, value in [("WEBHOOK_SECRET", "w"), ("SEARCH_API_KEY", "a"), ("SYNC_API_KEY", "s")]:
|
|
monkeypatch.setenv(name, value * 64)
|
|
data = tmp_path / "data"
|
|
snapshot(data, {"version": "old"})
|
|
state.durable_json(data / state.PENDING_FILE, {"target_version": "new"})
|
|
monkeypatch.setattr(routes, "DB_FILE", data / "zp_index.sqlite")
|
|
with TestClient(main.app) as client:
|
|
response = client.post(
|
|
"/rag", json={"query": "Aká práca?"}, headers={"Authorization": "Bearer " + "a" * 64}
|
|
)
|
|
assert response.status_code == 503
|
|
assert client.get("/health").json()["ready"] is False
|
|
|
|
|
|
@pytest.mark.parametrize("endpoint", ["sync", "webhook/gitea"])
|
|
def test_sync_api_exposes_common_version_and_graph_counts(monkeypatch, endpoint):
|
|
from fastapi.testclient import TestClient
|
|
import app.main as main
|
|
import app.routes as routes
|
|
|
|
monkeypatch.setattr(main, "embed_query", lambda q: None)
|
|
for name, value in [("WEBHOOK_SECRET", "w"), ("SEARCH_API_KEY", "a"), ("SYNC_API_KEY", "s")]:
|
|
monkeypatch.setenv(name, value * 64)
|
|
monkeypatch.setenv("EXPECTED_GITEA_REPOSITORY", "KEMT/zpwiki")
|
|
monkeypatch.setenv("WEBHOOK_PULL_GIT", "false")
|
|
version = {"version": "new", "git_revision": "commit"}
|
|
monkeypatch.setattr(
|
|
main,
|
|
"rebuild_index",
|
|
lambda **kwargs: {
|
|
"duration_seconds": 0.1,
|
|
"counts": {"documents": 1},
|
|
"index_version": version,
|
|
"graph_counts": {"nodes": 2, "relationships": 1},
|
|
},
|
|
)
|
|
with TestClient(main.app) as client:
|
|
if endpoint == "sync":
|
|
response = client.post("/sync", json={}, headers={"X-API-Key": "s" * 64})
|
|
else:
|
|
body = json.dumps(
|
|
{"repository": {"full_name": "KEMT/zpwiki"}, "ref": "refs/heads/master"}
|
|
).encode()
|
|
signature = hmac.new(("w" * 64).encode(), body, hashlib.sha256).hexdigest()
|
|
response = client.post(
|
|
"/webhook/gitea",
|
|
content=body,
|
|
headers={
|
|
"X-Gitea-Event": "push",
|
|
"X-Gitea-Signature": signature,
|
|
"Content-Type": "application/json",
|
|
},
|
|
)
|
|
assert response.status_code == 202
|
|
main.app.state.sync_jobs.run_once()
|
|
job = client.get(response.json()["status_url"], headers={"X-API-Key": "s" * 64}).json()
|
|
assert job["status"] == "succeeded"
|
|
assert job["result"]["index_version"] == version
|
|
assert job["result"]["graph_counts"] == {"nodes": 2, "relationships": 1}
|