146 lines
6.4 KiB
Python
146 lines
6.4 KiB
Python
"""Content is published with the source generation, including failure recovery."""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tools.knowledge import publication as content
|
|
from tools.knowledge import store
|
|
from tools.sync import index_state as state
|
|
from tools.sync import sync_indexes as sync
|
|
from test_index_sync import Session, snapshot
|
|
|
|
|
|
def checkpoint(version, label="content"):
|
|
return {"extraction_schema":2,"extraction_policy":"scoped-assertions-v2", "version": label, "index_version": version, "graph_ready": True,
|
|
"units": [], "profiles": 1}
|
|
|
|
|
|
class ContentSession(Session):
|
|
def __init__(self, version, content_state):
|
|
super().__init__(version)
|
|
self.content = content_state
|
|
|
|
def begin_transaction(self):
|
|
session = self
|
|
class Tx:
|
|
def __enter__(self):
|
|
self.version, self.content = session.version, session.content
|
|
return self
|
|
def __exit__(self, *args):
|
|
pass
|
|
def commit(self):
|
|
if session.fail_commit == "unknown":
|
|
session.unavailable = True
|
|
raise ConnectionError("unknown commit")
|
|
session.version, session.content = self.version, self.content
|
|
if session.fail_commit == "response_lost":
|
|
raise ConnectionError("lost response")
|
|
return Tx()
|
|
|
|
|
|
def graph_state(value):
|
|
return {"version": value["version"], "checkpoint_sha256": value["checkpoint_sha256"],
|
|
"index_version_json": json.dumps(value["index_version"], sort_keys=True)}
|
|
|
|
|
|
@pytest.fixture
|
|
def publication(tmp_path, monkeypatch):
|
|
old, new = {"version": "old"}, {"version": "new"}
|
|
data, stage = tmp_path / "data", tmp_path / "stage"
|
|
snapshot(data, old)
|
|
snapshot(stage, new)
|
|
old_state = content.checkpoint_state(data, checkpoint(old, "old-content"))
|
|
content.checkpoint_state(stage, checkpoint(new, "new-content"))
|
|
session = ContentSession(old, graph_state(old_state))
|
|
monkeypatch.setattr(sync, "write_graph_transaction", lambda tx, payload, version: setattr(tx, "version", version))
|
|
monkeypatch.setattr(sync, "graph_counts", lambda tx: ([("Person", 1)], []))
|
|
def write_content(tx, index, *, checkpoint_sha256=None):
|
|
tx.content = {"version": index["version"], "checkpoint_sha256": checkpoint_sha256,
|
|
"index_version_json": json.dumps(index["index_version"], sort_keys=True)}
|
|
monkeypatch.setattr(store, "write_graph_transaction", write_content)
|
|
def read_content(session):
|
|
if session.unavailable:
|
|
raise ConnectionError("database offline")
|
|
return session.content
|
|
monkeypatch.setattr(store, "graph_state", read_content)
|
|
return data, stage, old, new, session
|
|
|
|
|
|
@pytest.mark.parametrize("commit", [None, "response_lost"])
|
|
def test_publish_all_sources_and_content_in_same_transaction(publication, commit):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = commit
|
|
sync.publish_snapshot(data, stage, new, {}, session, knowledge=checkpoint(new, "new-content"))
|
|
assert state.read_version(data) == state.sqlite_version(data / "zp_index.sqlite") == session.version == new
|
|
assert store.load_index(data, new)["version"] == "new-content"
|
|
assert content.state_matches_graph(content.read_state(data), session.content)
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_failed_content_file_switch_restores_entire_generation(publication, monkeypatch):
|
|
data, stage, old, new, session = publication
|
|
files = (*sync.FILES, *content.FILES)
|
|
before = {f: (data / f).read_bytes() for f in files}
|
|
replace = sync.replace_file
|
|
def fail(source, target):
|
|
if target == data / "knowledge/state.json":
|
|
raise OSError("disk failed")
|
|
replace(source, target)
|
|
monkeypatch.setattr(sync, "replace_file", fail)
|
|
with pytest.raises(OSError):
|
|
sync.publish_snapshot(data, stage, new, {}, session, knowledge=checkpoint(new, "new-content"))
|
|
assert before == {f: (data / f).read_bytes() for f in files}
|
|
assert session.version == old
|
|
assert content.state_matches_graph(content.read_state(data), session.content)
|
|
|
|
|
|
def test_unknown_commit_blocks_metadata_and_content(publication):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = "unknown"
|
|
with pytest.raises(ConnectionError):
|
|
sync.publish_snapshot(data, stage, new, {}, session, knowledge=checkpoint(new, "new-content"))
|
|
with pytest.raises(state.IndexUnavailableError):
|
|
with state.index_snapshot(data):
|
|
pass
|
|
assert (stage / "previous/knowledge/index.json").exists()
|
|
|
|
|
|
@pytest.mark.parametrize("commit", [None, "response_lost", "unknown"])
|
|
def test_standalone_content_publication_failure_barrier(publication, commit):
|
|
data, stage, old, new, session = publication
|
|
session.fail_commit = commit
|
|
if commit == "unknown":
|
|
with pytest.raises(ConnectionError):
|
|
content.publish_checkpoint(data, stage, checkpoint(new, "new-content"), session)
|
|
with pytest.raises(state.IndexUnavailableError):
|
|
store.load_index(data, new)
|
|
assert (data / content.PENDING_FILE).exists()
|
|
else:
|
|
content.publish_checkpoint(data, stage, checkpoint(new, "new-content"), session)
|
|
assert content.state_matches_graph(content.read_state(data), session.content)
|
|
assert store.load_index(data, new)["version"] == "new-content"
|
|
|
|
|
|
def test_content_staging_failure_leaves_old_primary_and_content(publication, monkeypatch):
|
|
from tools.knowledge import build
|
|
data, stage, old, new, session = publication
|
|
before = {f: (data / f).read_bytes() for f in (*sync.FILES, *content.FILES)}
|
|
monkeypatch.setattr(sync, "prepare_snapshot", lambda *args: (new, {}, {}))
|
|
monkeypatch.setattr(build, "configured_extractor", lambda: (None, {}))
|
|
def fail(*args, **kwargs):
|
|
raise RuntimeError("LLM unavailable")
|
|
monkeypatch.setattr(build, "prepare_index", fail)
|
|
with pytest.raises(RuntimeError, match="LLM unavailable"):
|
|
sync.synchronize_indexes(Path("unused-wiki"), data)
|
|
assert before == {f: (data / f).read_bytes() for f in before}
|
|
assert not (data / state.PENDING_FILE).exists()
|
|
|
|
|
|
def test_corrupt_content_fails_integrity_check(publication):
|
|
data, *_ = publication
|
|
path = data / "knowledge/index.json"
|
|
path.write_text(path.read_text() + " ")
|
|
with pytest.raises(RuntimeError, match="invalid"):
|
|
store.load_index(data, {"version": "old"})
|