123 lines
4.9 KiB
Python
123 lines
4.9 KiB
Python
"""Recoverable publication of a content checkpoint and its Neo4j projection."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
|
|
from tools.sync.index_state import durable_json, fsync_directory, IndexUnavailableError
|
|
|
|
FILES = ("knowledge/index.json", "knowledge/state.json")
|
|
PENDING_FILE = "knowledge/.publication-pending.json"
|
|
|
|
|
|
def checkpoint_state(stage: Path, index: dict) -> dict:
|
|
directory = stage / "knowledge"
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
durable_json(directory / "index.json", index)
|
|
content = (directory / "index.json").read_bytes()
|
|
state = {
|
|
"schema": 1,
|
|
"extraction_schema": index.get("extraction_schema"),
|
|
"extraction_policy": index.get("extraction_policy"),
|
|
"coverage": index.get("coverage"),
|
|
"version": index["version"],
|
|
"index_version": index["index_version"],
|
|
"model": index.get("model", {}),
|
|
"graph_ready": index.get("graph_ready", False),
|
|
"embedding_fingerprint": index.get("embedding_fingerprint"),
|
|
"checkpoint_sha256": hashlib.sha256(content).hexdigest(),
|
|
"checkpoint_bytes": len(content),
|
|
"profiles": index.get("profiles"),
|
|
"units": len(index["units"]),
|
|
}
|
|
durable_json(directory / "state.json", state)
|
|
return state
|
|
|
|
|
|
def read_state(data_dir: Path) -> dict | None:
|
|
from tools.sync.index_state import PENDING_FILE as SOURCE_PENDING
|
|
if (data_dir / PENDING_FILE).exists() or (data_dir / SOURCE_PENDING).exists():
|
|
raise IndexUnavailableError("Publikovanie obsahovej extrakcie nebolo dokončené.")
|
|
path = data_dir / "knowledge/state.json"
|
|
return json.loads(path.read_text()) if path.exists() else None
|
|
|
|
|
|
def state_matches_graph(state: dict, actual: dict | None) -> bool:
|
|
return bool(actual and actual.get("version") == state["version"]
|
|
and actual.get("checkpoint_sha256") == state["checkpoint_sha256"]
|
|
and actual.get("index_version_json") == json.dumps(state["index_version"], sort_keys=True))
|
|
|
|
|
|
def replace_file(source: Path, target: Path) -> None:
|
|
os.replace(source, target)
|
|
|
|
|
|
def publish_checkpoint(data_dir: Path, stage: Path, index: dict, session=None) -> None:
|
|
"""Caller holds the exclusive publication lock and content build lock.
|
|
|
|
A failed/unknown commit leaves a durable marker; readers refuse that
|
|
checkpoint until a successful rebuild. A known rollback restores both files.
|
|
"""
|
|
from tools.knowledge.store import graph_state, write_graph_transaction
|
|
|
|
state = json.loads((stage / "knowledge/state.json").read_text())
|
|
old_graph = graph_state(session) if session is not None else None
|
|
backup = stage / "previous/knowledge"
|
|
backup.mkdir(parents=True)
|
|
originals = []
|
|
for name in FILES:
|
|
old = data_dir / name
|
|
if old.exists():
|
|
target = stage / "previous" / name
|
|
shutil.copy2(old, target)
|
|
with target.open("rb") as stream:
|
|
os.fsync(stream.fileno())
|
|
originals.append(name)
|
|
fsync_directory(backup)
|
|
had_pending = (data_dir / PENDING_FILE).exists()
|
|
durable_json(data_dir / PENDING_FILE,
|
|
{"staging_directory": stage.name, "target_version": index["version"]})
|
|
commit_attempted = False
|
|
try:
|
|
if session is None:
|
|
for name in FILES:
|
|
replace_file(stage / name, data_dir / name)
|
|
fsync_directory(data_dir / "knowledge")
|
|
else:
|
|
with session.begin_transaction() as tx:
|
|
write_graph_transaction(tx, index, checkpoint_sha256=state["checkpoint_sha256"])
|
|
for name in FILES:
|
|
replace_file(stage / name, data_dir / name)
|
|
fsync_directory(data_dir / "knowledge")
|
|
commit_attempted = True
|
|
tx.commit()
|
|
except BaseException:
|
|
if commit_attempted:
|
|
# Never equate a lost acknowledgement with a rolled-back commit.
|
|
actual = graph_state(session)
|
|
if state_matches_graph(state, actual):
|
|
if json.loads((data_dir / "knowledge/state.json").read_text()) != state:
|
|
raise
|
|
if hashlib.sha256((data_dir / "knowledge/index.json").read_bytes()).hexdigest() != state["checkpoint_sha256"]:
|
|
raise
|
|
(data_dir / PENDING_FILE).unlink()
|
|
fsync_directory(data_dir / "knowledge")
|
|
return
|
|
if actual != old_graph:
|
|
raise
|
|
for name in FILES:
|
|
if name in originals:
|
|
replace_file(stage / "previous" / name, data_dir / name)
|
|
else:
|
|
(data_dir / name).unlink(missing_ok=True)
|
|
fsync_directory(data_dir / "knowledge")
|
|
if not had_pending:
|
|
(data_dir / PENDING_FILE).unlink()
|
|
fsync_directory(data_dir / "knowledge")
|
|
raise
|
|
(data_dir / PENDING_FILE).unlink()
|
|
fsync_directory(data_dir / "knowledge")
|