"""Separate, version-bound Neo4j content graph; never clears the metadata graph.""" from __future__ import annotations import json from pathlib import Path from tools.knowledge.sources import identifier from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings from tools.answers.context_policy import normalized SCOPE = "zpwiki-content-v1" def create_constraints(session) -> None: for name, label, field in [ ("evidence", "KnowledgeEvidence", "id"), ("concept", "KnowledgeConcept", "id"), ("person", "KnowledgePerson", "source_path"), ("work", "KnowledgeWork", "id"), ("advisor", "KnowledgeAdvisor", "id"), ]: session.run( f"CREATE CONSTRAINT zp_content_{name} IF NOT EXISTS " f"FOR (n:{label}) REQUIRE (n.graph_scope,n.{field}) IS UNIQUE" ).consume() session.run( "CREATE CONSTRAINT zp_content_state IF NOT EXISTS " "FOR (n:KnowledgeState) REQUIRE n.scope IS UNIQUE" ).consume() def write_graph_transaction(tx, index: dict, *, checkpoint_sha256: str | None = None) -> None: """The caller owns the transaction and filesystem publication barrier.""" rows = [] for unit in index["units"]: rows.append( { **{k: v for k, v in unit.items() if k not in {"concepts", "relations"}}, "concepts": [ {"id": identifier(normalized(c)), "name": c} for c in unit["concepts"] ], "relations_json": json.dumps(unit["relations"], ensure_ascii=False), } ) tx.run( "MATCH (n:KnowledgeRelation {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE ).consume() tx.run( "MATCH (n:KnowledgeEvidence {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE ).consume() tx.run( "MATCH (n:KnowledgeConcept {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE ).consume() tx.run( "MATCH (n:KnowledgePerson {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE ).consume() evidence_rows = [ { "evidence": {k: v for k, v in row.items() if k != "concepts"} | {"graph_scope": SCOPE}, "concepts": row["concepts"], } for row in rows ] tx.run( """UNWIND $rows AS row CREATE (e:KnowledgeEvidence) SET e=row.evidence MERGE (p:KnowledgePerson {source_path:row.evidence.source_path,graph_scope:$scope}) SET p.name=row.evidence.person MERGE (p)-[:HAS_EVIDENCE]->(e) FOREACH (concept IN row.concepts | MERGE (c:KnowledgeConcept {id:concept.id,graph_scope:$scope}) SET c.name=concept.name MERGE (e)-[:ABOUT]->(c))""", rows=evidence_rows, scope=SCOPE, ).consume() relations = [ { "left": identifier(normalized(r["subject"])), "right": identifier(normalized(r["object"])), "evidence": row["id"], "predicate": r["predicate"], } for row in rows for r in json.loads(row["relations_json"]) ] tx.run( """UNWIND $rows AS row MATCH (a:KnowledgeConcept {id:row.left,graph_scope:$scope}), (b:KnowledgeConcept {id:row.right,graph_scope:$scope}), (e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope}) CREATE (r:KnowledgeRelation {graph_scope:$scope,predicate:row.predicate,origin:'llm'}) CREATE (r)-[:SUBJECT]->(a), (r)-[:OBJECT]->(b), (r)-[:EVIDENCED_BY]->(e)""", rows=relations, scope=SCOPE, ).consume() from tools.knowledge.sources import advisor_name tx.run( "MATCH (n:KnowledgeWork {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE ).consume() tx.run( "MATCH (n:KnowledgeAdvisor {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE ).consume() works = [ { "id": identifier(u["source_path"], u["work_label"]), **{ k: u.get(k) for k in ("source_path", "work_label", "work_type", "work_year") }, "evidence": u["id"], } for u in index["units"] if u["kind"] == "work_record" ] tx.run( """UNWIND $rows AS row MATCH (p:KnowledgePerson {source_path:row.source_path,graph_scope:$scope}), (e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope}) MERGE (w:KnowledgeWork {id:row.id,graph_scope:$scope}) SET w.label=row.work_label,w.type=row.work_type,w.year=row.work_year MERGE (p)-[:HAS_WORK]->(w) MERGE (w)-[:EVIDENCED_BY]->(e)""", rows=works, scope=SCOPE, ).consume() for role, edge in [("supervisor", "SUPERVISED_BY"), ("consultant", "CONSULTED_BY")]: advisors = [ { "work": identifier(u["source_path"], u["work_label"]), "id": identifier(normalized(name)), "name": name, "evidence": u["id"], } for u in index["units"] if u.get("work_label") and (name := advisor_name(u, role)) ] tx.run( f"""UNWIND $rows AS row MATCH (w:KnowledgeWork {{id:row.work,graph_scope:$scope}}), (e:KnowledgeEvidence {{id:row.evidence,graph_scope:$scope}}) MERGE (a:KnowledgeAdvisor {{id:row.id,graph_scope:$scope}}) SET a.name=row.name MERGE (w)-[:{edge}]->(a) MERGE (e)-[:DOCUMENTS_ROLE]->(a)""", rows=advisors, scope=SCOPE, ).consume() tx.run( """MATCH (e:KnowledgeEvidence {graph_scope:$scope}) WHERE e.meeting_id IS NOT NULL AND e.kind <> 'meeting' MATCH (m:KnowledgeEvidence {graph_scope:$scope,kind:'meeting',meeting_id:e.meeting_id}) MERGE (e)-[:RECORDED_AT]->(m)""", scope=SCOPE, ).consume() tx.run( """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w), (p)-[:HAS_EVIDENCE]->(e) WHERE e.work_label=w.label AND e.kind='meeting' MERGE (w)-[:HAS_MEETING]->(e)""", scope=SCOPE, ).consume() tx.run( """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w), (p)-[:HAS_EVIDENCE]->(e) WHERE e.work_label=w.label AND e.completion_status IS NOT NULL MERGE (w)-[:HAS_STATUS_RECORD]->(e)""", scope=SCOPE, ).consume() tx.run( "MERGE (s:KnowledgeState {scope:$scope}) SET s.version=$version, s.index_version_json=$index_version_json, s.checkpoint_sha256=$checkpoint_sha256", scope=SCOPE, version=index["version"], index_version_json=json.dumps(index.get("index_version"), sort_keys=True), checkpoint_sha256=checkpoint_sha256, ).consume() def graph_state(session) -> dict | None: record = session.run( "MATCH (s:KnowledgeState {scope:$scope}) RETURN properties(s) AS state", scope=SCOPE ).single() return dict(record["state"]) if record else None def write_graph(index: dict) -> None: """Low-level graph import; the production builder publishes through a journal.""" settings = get_neo4j_settings() with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: driver.verify_connectivity() create_constraints(session) with session.begin_transaction() as tx: write_graph_transaction(tx, index) tx.commit() def read_graph(index: dict) -> list[dict]: settings = get_neo4j_settings() with ( create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session, ): with session.begin_transaction() as tx: state = tx.run( "MATCH (s:KnowledgeState {scope:$scope}) RETURN s.version AS version, " "s.checkpoint_sha256 AS checkpoint_sha256", scope=SCOPE ).single() if not state or state["version"] != index["version"]: raise RuntimeError("Content graph and extraction checkpoint differ.") if index.get("_checkpoint_sha256") and state["checkpoint_sha256"] != index["_checkpoint_sha256"]: raise RuntimeError("Content graph and checkpoint checksum differ.") rows = tx.run( """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_EVIDENCE]->(e) OPTIONAL MATCH (e)-[:ABOUT]->(c:KnowledgeConcept) RETURN properties(e) AS evidence, collect(c.name) AS concepts ORDER BY evidence.source_path,evidence.line""", scope=SCOPE, ) result = [{**dict(r["evidence"]), "concepts": list(r["concepts"])} for r in rows] tx.commit() return result def load_index(data_dir: Path, version: dict | None) -> dict | None: from tools.knowledge.publication import read_state import hashlib state = read_state(data_dir) path = data_dir / "knowledge/index.json" if not path.exists(): return None content = path.read_bytes() index = json.loads(content) if not version or index.get("index_version") != version: raise RuntimeError("Content extraction belongs to another source snapshot. Rebuild it.") if state is not None: if not isinstance(state, dict) or state.get("schema") != 1: raise RuntimeError("Content extraction publication state is invalid.") if (state.get("version") != index.get("version") or state.get("index_version") != version or hashlib.sha256(content).hexdigest() != state.get("checkpoint_sha256")): raise RuntimeError("Content extraction checkpoint is invalid. Rebuild it.") index["_checkpoint_sha256"] = state["checkpoint_sha256"] return index