287 lines
13 KiB
Python
287 lines
13 KiB
Python
"""Separate, version-bound Neo4j content graph; never clears the metadata graph."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from tools.knowledge.sources import identifier
|
|
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
|
from tools.answers.context_policy import normalized
|
|
from tools.knowledge.concepts import concept_key
|
|
|
|
SCOPE = "zpwiki-content-v1"
|
|
CONTENT_GRAPH_SCHEMA = 2
|
|
|
|
|
|
def create_constraints(session) -> None:
|
|
for name, label, field in [
|
|
("evidence", "KnowledgeEvidence", "id"),
|
|
("concept", "KnowledgeConcept", "id"),
|
|
("person", "KnowledgePerson", "source_path"),
|
|
("work", "KnowledgeWork", "id"),
|
|
("advisor", "KnowledgeAdvisor", "id"),
|
|
("activity", "KnowledgeActivity", "id"),
|
|
]:
|
|
session.run(
|
|
f"CREATE CONSTRAINT zp_content_{name} IF NOT EXISTS "
|
|
f"FOR (n:{label}) REQUIRE (n.graph_scope,n.{field}) IS UNIQUE"
|
|
).consume()
|
|
session.run(
|
|
"CREATE CONSTRAINT zp_content_state IF NOT EXISTS "
|
|
"FOR (n:KnowledgeState) REQUIRE n.scope IS UNIQUE"
|
|
).consume()
|
|
session.run('CREATE INDEX zp_content_concept_key IF NOT EXISTS '
|
|
'FOR (c:KnowledgeConcept) ON (c.graph_scope,c.key)').consume()
|
|
|
|
|
|
def write_graph_transaction(tx, index: dict, *, checkpoint_sha256: str | None = None) -> None:
|
|
"""The caller owns the transaction and filesystem publication barrier."""
|
|
from tools.knowledge.evidence import EXTRACTION_SCHEMA, relation_has_local_support, claim_eligible
|
|
rows = []
|
|
for unit in index["units"]:
|
|
if index.get("extraction_schema") == EXTRACTION_SCHEMA and any(
|
|
r.get("verification") != "supported" or not relation_has_local_support(unit,r)
|
|
for r in unit.get("relations",[])
|
|
):
|
|
raise ValueError("Unverified relationship cannot enter the active content graph")
|
|
rows.append(
|
|
{
|
|
**{k: v for k, v in unit.items() if k not in {"concepts", "relations"}},
|
|
"concepts": [
|
|
{"id": identifier(normalized(c)), "name": c, 'key': concept_key(c)} for c in unit["concepts"]
|
|
],
|
|
"relations_json": json.dumps(unit["relations"], ensure_ascii=False),
|
|
}
|
|
)
|
|
tx.run(
|
|
"MATCH (n:KnowledgeActivity {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
tx.run(
|
|
"MATCH (n:KnowledgeRelation {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
tx.run(
|
|
"MATCH (n:KnowledgeEvidence {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
tx.run(
|
|
"MATCH (n:KnowledgeConcept {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
tx.run(
|
|
"MATCH (n:KnowledgePerson {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
evidence_rows = [
|
|
{
|
|
"evidence": {k: v for k, v in row.items() if k != "concepts"}
|
|
| {"graph_scope": SCOPE},
|
|
"concepts": row["concepts"],
|
|
}
|
|
for row in rows
|
|
]
|
|
tx.run(
|
|
"""UNWIND $rows AS row
|
|
CREATE (e:KnowledgeEvidence) SET e=row.evidence
|
|
MERGE (p:KnowledgePerson {source_path:row.evidence.source_path,graph_scope:$scope})
|
|
SET p.name=row.evidence.person
|
|
MERGE (p)-[:HAS_EVIDENCE]->(e)
|
|
FOREACH (concept IN row.concepts |
|
|
MERGE (c:KnowledgeConcept {id:concept.id,graph_scope:$scope})
|
|
SET c.name=concept.name,c.key=concept.key MERGE (e)-[:ABOUT]->(c))""",
|
|
rows=evidence_rows,
|
|
scope=SCOPE,
|
|
).consume()
|
|
relations = [
|
|
{
|
|
"left": identifier(normalized(r["subject"])),
|
|
"right": identifier(normalized(r["object"])),
|
|
"evidence": row["id"],
|
|
"predicate": r["predicate"],
|
|
"quote": r.get("quote"),
|
|
"verification": r.get("verification"),
|
|
**{k:row.get(k) for k in ("actor","actor_status","work_id","work_label","work_type","work_year","phase","section_scope","record_date","event_date","event_year","event_date_quote","target_date","target_year","source_path","source_url","line")},
|
|
}
|
|
for row in rows
|
|
for r in json.loads(row["relations_json"])
|
|
]
|
|
tx.run(
|
|
"""UNWIND $rows AS row
|
|
MATCH (a:KnowledgeConcept {id:row.left,graph_scope:$scope}),
|
|
(b:KnowledgeConcept {id:row.right,graph_scope:$scope}),
|
|
(e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope})
|
|
CREATE (r:KnowledgeRelation {graph_scope:$scope,origin:$origin})
|
|
SET r += row
|
|
CREATE (r)-[:SUBJECT]->(a), (r)-[:OBJECT]->(b), (r)-[:EVIDENCED_BY]->(e)""",
|
|
rows=relations,
|
|
scope=SCOPE,
|
|
origin="llm_checked" if index.get("extraction_schema") == EXTRACTION_SCHEMA else "legacy_unverified",
|
|
).consume()
|
|
from tools.knowledge.sources import advisor_name
|
|
tx.run(
|
|
"MATCH (n:KnowledgeWork {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
tx.run(
|
|
"MATCH (n:KnowledgeAdvisor {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE
|
|
).consume()
|
|
works = [
|
|
{
|
|
"id": identifier(u["source_path"], u["work_label"]),
|
|
**{
|
|
k: u.get(k) for k in ("source_path", "work_label", "work_type", "work_year")
|
|
},
|
|
"evidence": u["id"],
|
|
}
|
|
for u in index["units"]
|
|
if u["kind"] == "work_record"
|
|
]
|
|
tx.run(
|
|
"""UNWIND $rows AS row
|
|
MATCH (p:KnowledgePerson {source_path:row.source_path,graph_scope:$scope}),
|
|
(e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope})
|
|
MERGE (w:KnowledgeWork {id:row.id,graph_scope:$scope})
|
|
SET w.label=row.work_label,w.type=row.work_type,w.year=row.work_year,w.source_path=row.source_path
|
|
MERGE (p)-[:HAS_WORK]->(w) MERGE (w)-[:EVIDENCED_BY]->(e)""",
|
|
rows=works,
|
|
scope=SCOPE,
|
|
).consume()
|
|
supported = [u for u in index['units'] if u.get('work_id') and claim_eligible(u)]
|
|
tx.run('''UNWIND $rows AS row
|
|
MATCH (w:KnowledgeWork {id:row.work_id,graph_scope:$scope}),
|
|
(e:KnowledgeEvidence {id:row.id,graph_scope:$scope})
|
|
WHERE w.source_path=e.source_path AND w.label=e.work_label
|
|
MERGE (w)-[:EVIDENCED_BY]->(e)''', rows=supported, scope=SCOPE).consume()
|
|
activities = [u for u in supported if u.get('annotation_status') == 'knowledge'
|
|
and u.get('actor') == u.get('person')
|
|
and u.get('actor_status') in {'explicit_owner','profile_context'}
|
|
and u.get('kind') in {'assignment','plan','progress','result'}]
|
|
tx.run('''UNWIND $rows AS row
|
|
MATCH (w:KnowledgeWork {id:row.work_id,graph_scope:$scope}),
|
|
(e:KnowledgeEvidence {id:row.id,graph_scope:$scope})
|
|
WHERE w.source_path=e.source_path AND w.label=e.work_label
|
|
CREATE (a:KnowledgeActivity {graph_scope:$scope,id:row.id,
|
|
work_id:row.work_id,kind:row.kind,phase:row.phase})
|
|
CREATE (w)-[:HAS_ACTIVITY]->(a), (a)-[:SUPPORTED_BY]->(e)''',
|
|
rows=activities, scope=SCOPE).consume()
|
|
for role, edge in [("supervisor", "SUPERVISED_BY"), ("consultant", "CONSULTED_BY")]:
|
|
advisors = [
|
|
{
|
|
"work": identifier(u["source_path"], u["work_label"]),
|
|
"id": identifier(normalized(name)),
|
|
"name": name,
|
|
"evidence": u["id"],
|
|
}
|
|
for u in index["units"]
|
|
if u.get("work_label") and (name := advisor_name(u, role))
|
|
]
|
|
tx.run(
|
|
f"""UNWIND $rows AS row
|
|
MATCH (w:KnowledgeWork {{id:row.work,graph_scope:$scope}}),
|
|
(e:KnowledgeEvidence {{id:row.evidence,graph_scope:$scope}})
|
|
MERGE (a:KnowledgeAdvisor {{id:row.id,graph_scope:$scope}}) SET a.name=row.name
|
|
MERGE (w)-[:{edge}]->(a) MERGE (e)-[:DOCUMENTS_ROLE]->(a)""",
|
|
rows=advisors,
|
|
scope=SCOPE,
|
|
).consume()
|
|
tx.run(
|
|
"""MATCH (e:KnowledgeEvidence {graph_scope:$scope})
|
|
WHERE e.meeting_id IS NOT NULL AND e.kind <> 'meeting'
|
|
MATCH (m:KnowledgeEvidence {graph_scope:$scope,kind:'meeting',meeting_id:e.meeting_id})
|
|
MERGE (e)-[:RECORDED_AT]->(m)""",
|
|
scope=SCOPE,
|
|
).consume()
|
|
tx.run(
|
|
"""MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w),
|
|
(p)-[:HAS_EVIDENCE]->(e)
|
|
WHERE e.work_label=w.label AND e.kind='meeting'
|
|
MERGE (w)-[:HAS_MEETING]->(e)""",
|
|
scope=SCOPE,
|
|
).consume()
|
|
tx.run(
|
|
"""MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w),
|
|
(p)-[:HAS_EVIDENCE]->(e)
|
|
WHERE e.work_label=w.label AND e.completion_status IS NOT NULL
|
|
MERGE (w)-[:HAS_STATUS_RECORD]->(e)""",
|
|
scope=SCOPE,
|
|
).consume()
|
|
tx.run(
|
|
"""MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_EVIDENCE]->(e)
|
|
WHERE e.actor=p.name AND e.actor_status IN ['explicit_owner','profile_context']
|
|
AND e.annotation_status IN ['knowledge','structural']
|
|
MERGE (e)-[r:ATTRIBUTED_TO]->(p) SET r.basis=e.actor_status""", scope=SCOPE
|
|
).consume()
|
|
tx.run(
|
|
"MERGE (s:KnowledgeState {scope:$scope}) SET s.version=$version, s.index_version_json=$index_version_json, s.checkpoint_sha256=$checkpoint_sha256,s.projection_schema=$projection_schema",
|
|
scope=SCOPE,
|
|
version=index["version"],
|
|
index_version_json=json.dumps(index.get("index_version"), sort_keys=True),
|
|
checkpoint_sha256=checkpoint_sha256,
|
|
projection_schema=CONTENT_GRAPH_SCHEMA,
|
|
).consume()
|
|
|
|
|
|
def graph_state(session) -> dict | None:
|
|
record = session.run(
|
|
"MATCH (s:KnowledgeState {scope:$scope}) RETURN properties(s) AS state", scope=SCOPE
|
|
).single()
|
|
return dict(record["state"]) if record else None
|
|
|
|
|
|
def write_graph(index: dict) -> None:
|
|
"""Low-level graph import; the production builder publishes through a journal."""
|
|
settings = get_neo4j_settings()
|
|
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
|
|
driver.verify_connectivity()
|
|
create_constraints(session)
|
|
with session.begin_transaction() as tx:
|
|
write_graph_transaction(tx, index)
|
|
tx.commit()
|
|
|
|
|
|
|
|
def read_graph(index: dict) -> list[dict]:
|
|
settings = get_neo4j_settings()
|
|
with (
|
|
create_neo4j_driver(settings) as driver,
|
|
driver.session(database=settings.database) as session,
|
|
):
|
|
with session.begin_transaction() as tx:
|
|
state = tx.run(
|
|
"MATCH (s:KnowledgeState {scope:$scope}) RETURN s.version AS version, "
|
|
"s.checkpoint_sha256 AS checkpoint_sha256", scope=SCOPE
|
|
).single()
|
|
if not state or state["version"] != index["version"]:
|
|
raise RuntimeError("Content graph and extraction checkpoint differ.")
|
|
if index.get("_checkpoint_sha256") and state["checkpoint_sha256"] != index["_checkpoint_sha256"]:
|
|
raise RuntimeError("Content graph and checkpoint checksum differ.")
|
|
rows = tx.run(
|
|
"""MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_EVIDENCE]->(e)
|
|
OPTIONAL MATCH (e)-[:ABOUT]->(c:KnowledgeConcept)
|
|
RETURN properties(e) AS evidence, collect(c.name) AS concepts ORDER BY evidence.source_path,evidence.line""",
|
|
scope=SCOPE,
|
|
)
|
|
result = [{**dict(r["evidence"]), "concepts": list(r["concepts"])} for r in rows]
|
|
tx.commit()
|
|
return result
|
|
|
|
|
|
def load_index(data_dir: Path, version: dict | None) -> dict | None:
|
|
from tools.knowledge.publication import read_state
|
|
import hashlib
|
|
|
|
state = read_state(data_dir)
|
|
path = data_dir / "knowledge/index.json"
|
|
if not path.exists():
|
|
return None
|
|
content = path.read_bytes()
|
|
index = json.loads(content)
|
|
if not version or index.get("index_version") != version:
|
|
raise RuntimeError("Content extraction belongs to another source snapshot. Rebuild it.")
|
|
if state is not None:
|
|
if not isinstance(state, dict) or state.get("schema") != 1:
|
|
raise RuntimeError("Content extraction publication state is invalid.")
|
|
if (state.get("version") != index.get("version") or state.get("index_version") != version
|
|
or hashlib.sha256(content).hexdigest() != state.get("checkpoint_sha256")):
|
|
raise RuntimeError("Content extraction checkpoint is invalid. Rebuild it.")
|
|
index["_checkpoint_sha256"] = state["checkpoint_sha256"]
|
|
from tools.knowledge.evidence import EXTRACTION_SCHEMA, EXTRACTION_POLICY
|
|
if index.get("extraction_schema") != EXTRACTION_SCHEMA or index.get("extraction_policy") != EXTRACTION_POLICY:
|
|
raise RuntimeError("Content extraction policy changed; rebuild the checkpoint before serving it.")
|
|
return index
|