"""Stage SQLite/JSON and publish them with a single Neo4j data transaction. There is no distributed transaction between a filesystem and Neo4j. Cooperating readers therefore use the publication barrier. Ordinary failures restore local files; ambiguous commit/crash leaves a durable marker and readers fail closed. Re-running sync creates a complete fresh generation and clears that marker. """ from __future__ import annotations import hashlib import json import os import shutil import sqlite3 import subprocess import tempfile import uuid from contextlib import closing, ExitStack from datetime import datetime, timezone from pathlib import Path from tools.sync.index_state import ( PENDING_FILE, VERSION_FILE, durable_json, fsync_directory, publication_lock, sqlite_version, ) from tools.rag.scan_zpwiki import scan_pages from tools.rag.build_chunks import build_chunks from tools.rag.build_sqlite_index import build_database from tools.rag.embedding_utils import embedding_model_name from tools.retrieval.publication import PUBLICATION_POLICY_VERSION from tools.graphrag.graphrag import build_graph_payload from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings from tools.graphrag.build_graphrag import ( create_constraints, graph_version, write_graph_transaction, graph_counts, ) FILES = ("documents.json", "chunks.json", "zp_index.sqlite", VERSION_FILE) def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def source_files(wiki_root: Path) -> dict[str, str]: if not (wiki_root / "pages").is_dir(): raise RuntimeError(f"Chýba wiki priečinok pages: {wiki_root}") return { str(p.relative_to(wiki_root)): digest(p) for p in sorted((wiki_root / "pages").rglob("README.md")) } def git_revision(wiki_root: Path) -> str | None: if not (wiki_root / ".git").exists(): return None result = subprocess.run( ["git", "-C", str(wiki_root), "rev-parse", "HEAD"], capture_output=True, text=True, timeout=15, check=True, ) return result.stdout.strip() def prepare_snapshot(wiki_root: Path, stage: Path) -> tuple[dict, dict, dict]: revision = git_revision(wiki_root) sources = source_files(wiki_root) frozen_wiki = stage / "wiki" for relative in sources: target = frozen_wiki / relative target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(wiki_root / relative, target) if source_files(frozen_wiki) != sources: raise RuntimeError("Wiki sa počas vytvárania snapshotu zmenila.") documents = scan_pages(wiki_root=frozen_wiki, output=stage / "documents.json") chunks = build_chunks(wiki_root=frozen_wiki, output=stage / "chunks.json") counts = build_database( documents_file=stage / "documents.json", chunks_file=stage / "chunks.json", db_file=stage / "zp_index.sqlite", cache_file=stage.parent / "embedding_cache.sqlite", ) version = { "version": uuid.uuid4().hex, "git_revision": revision, "created_at_utc": datetime.now(timezone.utc).isoformat(), "source_sha256": hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(), "documents_sha256": digest(stage / "documents.json"), "chunks_sha256": digest(stage / "chunks.json"), "embedding_model": embedding_model_name(), "publication_policy": PUBLICATION_POLICY_VERSION, } if chunks: from tools.rag.embedding_utils import model_profile version["embedding_fingerprint"] = model_profile(embedding_model_name())["fingerprint"] with closing(sqlite3.connect(stage / "zp_index.sqlite")) as conn, conn: conn.execute( "CREATE TABLE index_state (id INTEGER PRIMARY KEY CHECK(id=1), metadata_json TEXT NOT NULL)" ) conn.execute( "INSERT INTO index_state VALUES (1, ?)", (json.dumps(version, sort_keys=True),) ) durable_json(stage / VERSION_FILE, version) if source_files(wiki_root) != sources or git_revision(wiki_root) != revision: raise RuntimeError("Wiki sa počas indexovania zmenila; pripravené indexy sa neaktivujú.") return version, counts, build_graph_payload(documents, chunks) def replace_file(source: Path, target: Path) -> None: """Separate operation so tests can inject a publication I/O failure.""" os.replace(source, target) def publish_snapshot(data_dir: Path, stage: Path, version: dict, payload: dict, session, *, knowledge=None) -> dict: with publication_lock(data_dir, exclusive=True): from tools.knowledge import publication as content_publication from tools.knowledge import store as content_store files = (*FILES, *content_publication.FILES) if knowledge is not None else FILES content_state = (json.loads((stage / "knowledge/state.json").read_text()) if knowledge is not None else None) had_pending = (data_dir / PENDING_FILE).exists() old_graph_version = graph_version(session) originals = [] backup = stage / "previous" backup.mkdir() for name in files: current = data_dir / name if current.exists(): (backup / name).parent.mkdir(parents=True, exist_ok=True) shutil.copy2(current, backup / name) with (backup / name).open("rb") as handle: os.fsync(handle.fileno()) originals.append(name) for directory in {backup, *((backup / name).parent for name in originals)}: fsync_directory(directory) durable_json( data_dir / PENDING_FILE, { "target_version": version["version"], "staging_directory": stage.name, "original_files": originals, }, ) commit_attempted = False try: with session.begin_transaction() as tx: write_graph_transaction(tx, payload, version=version) if knowledge is not None: content_store.write_graph_transaction(tx, knowledge, checkpoint_sha256=content_state["checkpoint_sha256"]) node_counts, relation_counts = graph_counts(tx) graph_summary = { "nodes": sum(n for _, n in node_counts), "relationships": sum(n for _, n in relation_counts), } for name in files: # Flush staged contents before the durable directory switch. with (stage / name).open("rb") as handle: os.fsync(handle.fileno()) (data_dir / name).parent.mkdir(parents=True, exist_ok=True) replace_file(stage / name, data_dir / name) for parent in { (data_dir / name).parent for name in files }: fsync_directory(parent) commit_attempted = True tx.commit() except BaseException: if commit_attempted: # A lost COMMIT response is not proof of rollback. Never restore # old SQLite against a possibly committed new graph. try: current_graph_version = graph_version(session) except Exception: raise # Keep marker + backups; reads stay unavailable. if ( current_graph_version == version and sqlite_version(data_dir / "zp_index.sqlite") == version ): if knowledge is not None: if not content_publication.state_matches_graph(content_state, content_store.graph_state(session)): raise if (json.loads((data_dir / "knowledge/state.json").read_text()) != content_state or digest(data_dir / "knowledge/index.json") != content_state["checkpoint_sha256"]): raise (data_dir / PENDING_FILE).unlink() if knowledge is not None: (data_dir / content_publication.PENDING_FILE).unlink(missing_ok=True) fsync_directory(data_dir / 'knowledge') fsync_directory(data_dir) return graph_summary if current_graph_version != old_graph_version: raise # Unknown external writer; fail closed. for name in files: if name in originals: os.replace(backup / name, data_dir / name) else: (data_dir / name).unlink(missing_ok=True) for directory in {(data_dir / name).parent for name in files}: fsync_directory(directory) if not had_pending: (data_dir / PENDING_FILE).unlink() fsync_directory(data_dir) raise (data_dir / PENDING_FILE).unlink() if knowledge is not None: (data_dir / content_publication.PENDING_FILE).unlink(missing_ok=True) fsync_directory(data_dir / 'knowledge') fsync_directory(data_dir) return graph_summary def synchronize_indexes(wiki_root: Path, data_dir: Path, *, progress=None) -> dict: data_dir.mkdir(parents=True, exist_ok=True) stage = Path(tempfile.mkdtemp(prefix=".sync-stage-", dir=data_dir)) try: version, counts, payload = prepare_snapshot(wiki_root, stage) knowledge = None knowledge_summary = None with ExitStack() as locks: if (data_dir / "knowledge/index.json").exists(): from tools.rag.rebuild_index import acquire_reindex_lock from tools.knowledge.build import prepare_index, configured_extractor from tools.knowledge.publication import checkpoint_state locks.enter_context(acquire_reindex_lock(data_dir / "knowledge/.build.lock")) if progress: progress("extracting_knowledge") complete, identity = configured_extractor() knowledge, knowledge_summary = prepare_index(stage / 'wiki', data_dir, complete, identity, source_version=version) checkpoint_state(stage, knowledge) if git_revision(wiki_root) != version['git_revision'] or hashlib.sha256( json.dumps(source_files(wiki_root),sort_keys=True).encode()).hexdigest() != version['source_sha256']: raise RuntimeError("Wiki sa počas extrakcie zmenila; generácia nebola aktivovaná.") graph_summary = _publish_prepared(data_dir, stage, version, payload, knowledge, progress) return { "counts": counts, "documents_scanned": counts["documents"], "chunks_created": counts["chunks"], "database_path": str(data_dir / "zp_index.sqlite"), "index_version": version, "graph_counts": graph_summary, **({"knowledge": knowledge_summary} if knowledge_summary is not None else {}), } finally: # Keep recovery material when a crash/ambiguous commit is unresolved. pending = data_dir / PENDING_FILE keep_stage = False if pending.exists(): try: keep_stage = json.loads(pending.read_text()).get("staging_directory") == stage.name except (ValueError, OSError): keep_stage = True if not keep_stage: shutil.rmtree(stage) def _publish_prepared(data_dir, stage, version, payload, knowledge, progress): if progress: progress("publishing_indexes") settings = get_neo4j_settings() with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: driver.verify_connectivity() create_constraints(session) if knowledge is not None: from tools.knowledge.store import create_constraints as create_content_constraints create_content_constraints(session) return publish_snapshot(data_dir, stage, version, payload, session, knowledge=knowledge)