120 lines
8.0 KiB
Python
120 lines
8.0 KiB
Python
"""Separate, version-bound Neo4j content graph; never clears the metadata graph."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from tools.knowledge.sources import identifier
|
|
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
|
from tools.answer_evaluation.context_policy import normalized
|
|
|
|
SCOPE = 'zpwiki-content-v1'
|
|
|
|
|
|
def write_graph(index: dict) -> None:
|
|
settings = get_neo4j_settings()
|
|
rows = []
|
|
for unit in index['units']:
|
|
rows.append({**{k: v for k, v in unit.items() if k not in {'concepts', 'relations'}},
|
|
'concepts': [{'id': identifier(normalized(c)), 'name': c} for c in unit['concepts']],
|
|
'relations_json': json.dumps(unit['relations'], ensure_ascii=False)})
|
|
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
|
|
driver.verify_connectivity()
|
|
for name,label,field in [('evidence','KnowledgeEvidence','id'),('concept','KnowledgeConcept','id'),
|
|
('person','KnowledgePerson','source_path'),
|
|
('work','KnowledgeWork','id'),('advisor','KnowledgeAdvisor','id')]:
|
|
session.run(f'CREATE CONSTRAINT zp_content_{name} IF NOT EXISTS '
|
|
f'FOR (n:{label}) REQUIRE (n.graph_scope,n.{field}) IS UNIQUE').consume()
|
|
session.run('CREATE CONSTRAINT zp_content_state IF NOT EXISTS '
|
|
'FOR (n:KnowledgeState) REQUIRE n.scope IS UNIQUE').consume()
|
|
with session.begin_transaction() as tx:
|
|
tx.run('MATCH (n:KnowledgeRelation {graph_scope:$scope}) DETACH DELETE n', scope=SCOPE).consume()
|
|
tx.run('MATCH (n:KnowledgeEvidence {graph_scope:$scope}) DETACH DELETE n', scope=SCOPE).consume()
|
|
tx.run('MATCH (n:KnowledgeConcept {graph_scope:$scope}) DETACH DELETE n', scope=SCOPE).consume()
|
|
tx.run('MATCH (n:KnowledgePerson {graph_scope:$scope}) DETACH DELETE n', scope=SCOPE).consume()
|
|
# Copy only scalar evidence properties. Dynamic labels/Cypher from
|
|
# the model are never executed; all relationships have fixed types.
|
|
evidence_rows = [{'evidence': {k:v for k,v in row.items() if k != 'concepts'} | {'graph_scope':SCOPE},
|
|
'concepts':row['concepts']} for row in rows]
|
|
tx.run('''UNWIND $rows AS row
|
|
CREATE (e:KnowledgeEvidence) SET e=row.evidence
|
|
MERGE (p:KnowledgePerson {source_path:row.evidence.source_path,graph_scope:$scope})
|
|
SET p.name=row.evidence.person
|
|
MERGE (p)-[:HAS_EVIDENCE]->(e)
|
|
FOREACH (concept IN row.concepts |
|
|
MERGE (c:KnowledgeConcept {id:concept.id,graph_scope:$scope})
|
|
SET c.name=concept.name MERGE (e)-[:ABOUT]->(c))''',rows=evidence_rows,scope=SCOPE).consume()
|
|
relations = [{'left':identifier(normalized(r['subject'])), 'right':identifier(normalized(r['object'])),
|
|
'evidence':row['id'], 'predicate':r['predicate']}
|
|
for row in rows for r in json.loads(row['relations_json'])]
|
|
tx.run('''UNWIND $rows AS row
|
|
MATCH (a:KnowledgeConcept {id:row.left,graph_scope:$scope}),
|
|
(b:KnowledgeConcept {id:row.right,graph_scope:$scope}),
|
|
(e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope})
|
|
CREATE (r:KnowledgeRelation {graph_scope:$scope,predicate:row.predicate,origin:'llm'})
|
|
CREATE (r)-[:SUBJECT]->(a), (r)-[:OBJECT]->(b), (r)-[:EVIDENCED_BY]->(e)''',
|
|
rows=relations,scope=SCOPE).consume()
|
|
from tools.knowledge.sources import advisor_name
|
|
# Roles are explicit, source-backed edges attached to an exact work.
|
|
# Profile-level roles remain unassigned; they are never spread over
|
|
# every work on that profile.
|
|
tx.run('MATCH (n:KnowledgeWork {graph_scope:$scope}) DETACH DELETE n',scope=SCOPE).consume()
|
|
tx.run('MATCH (n:KnowledgeAdvisor {graph_scope:$scope}) DETACH DELETE n',scope=SCOPE).consume()
|
|
works=[{'id':identifier(u['source_path'],u['work_label']),
|
|
**{k:u.get(k) for k in ('source_path','work_label','work_type','work_year')},'evidence':u['id']}
|
|
for u in index['units'] if u['kind']=='work_record']
|
|
tx.run('''UNWIND $rows AS row
|
|
MATCH (p:KnowledgePerson {source_path:row.source_path,graph_scope:$scope}),
|
|
(e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope})
|
|
MERGE (w:KnowledgeWork {id:row.id,graph_scope:$scope})
|
|
SET w.label=row.work_label,w.type=row.work_type,w.year=row.work_year
|
|
MERGE (p)-[:HAS_WORK]->(w) MERGE (w)-[:EVIDENCED_BY]->(e)''',rows=works,scope=SCOPE).consume()
|
|
for role,edge in [('supervisor','SUPERVISED_BY'),('consultant','CONSULTED_BY')]:
|
|
advisors=[{'work':identifier(u['source_path'],u['work_label']),
|
|
'id':identifier(normalized(name)),'name':name,'evidence':u['id']}
|
|
for u in index['units'] if u.get('work_label') and (name:=advisor_name(u,role))]
|
|
tx.run(f'''UNWIND $rows AS row
|
|
MATCH (w:KnowledgeWork {{id:row.work,graph_scope:$scope}}),
|
|
(e:KnowledgeEvidence {{id:row.evidence,graph_scope:$scope}})
|
|
MERGE (a:KnowledgeAdvisor {{id:row.id,graph_scope:$scope}}) SET a.name=row.name
|
|
MERGE (w)-[:{edge}]->(a) MERGE (e)-[:DOCUMENTS_ROLE]->(a)''',rows=advisors,scope=SCOPE).consume()
|
|
tx.run('''MATCH (e:KnowledgeEvidence {graph_scope:$scope})
|
|
WHERE e.meeting_id IS NOT NULL AND e.kind <> 'meeting'
|
|
MATCH (m:KnowledgeEvidence {graph_scope:$scope,kind:'meeting',meeting_id:e.meeting_id})
|
|
MERGE (e)-[:RECORDED_AT]->(m)''',scope=SCOPE).consume()
|
|
tx.run('''MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w),
|
|
(p)-[:HAS_EVIDENCE]->(e)
|
|
WHERE e.work_label=w.label AND e.kind='meeting'
|
|
MERGE (w)-[:HAS_MEETING]->(e)''',scope=SCOPE).consume()
|
|
tx.run('''MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w),
|
|
(p)-[:HAS_EVIDENCE]->(e)
|
|
WHERE e.work_label=w.label AND e.completion_status IS NOT NULL
|
|
MERGE (w)-[:HAS_STATUS_RECORD]->(e)''',scope=SCOPE).consume()
|
|
tx.run('MERGE (s:KnowledgeState {scope:$scope}) SET s.version=$version',
|
|
scope=SCOPE, version=index['version']).consume()
|
|
tx.commit()
|
|
|
|
|
|
def read_graph(index: dict) -> list[dict]:
|
|
settings = get_neo4j_settings()
|
|
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
|
|
with session.begin_transaction() as tx:
|
|
state = tx.run('MATCH (s:KnowledgeState {scope:$scope}) RETURN s.version AS version', scope=SCOPE).single()
|
|
if not state or state['version'] != index['version']:
|
|
raise RuntimeError('Content graph and extraction checkpoint differ.')
|
|
rows = tx.run('''MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_EVIDENCE]->(e)
|
|
OPTIONAL MATCH (e)-[:ABOUT]->(c:KnowledgeConcept)
|
|
RETURN properties(e) AS evidence, collect(c.name) AS concepts ORDER BY evidence.source_path,evidence.line''', scope=SCOPE)
|
|
result = [{**dict(r['evidence']), 'concepts': list(r['concepts'])} for r in rows]
|
|
tx.commit()
|
|
return result
|
|
|
|
|
|
def load_index(data_dir: Path, version: dict | None) -> dict | None:
|
|
path = data_dir / 'knowledge/index.json'
|
|
if not path.exists():
|
|
return None
|
|
index = json.loads(path.read_text())
|
|
if not version or index.get('index_version') != version:
|
|
raise RuntimeError('Content extraction belongs to another source snapshot. Rebuild it.')
|
|
return index
|