from __future__ import annotations import argparse import json from pathlib import Path from typing import Any from .graph_db import ( create_neo4j_driver, get_neo4j_settings, ) from .graphrag import ( GRAPH_SCOPE, build_graph_payload, ) from tools.sync.index_state import DATA_DIR, PENDING_FILE, publication_lock, read_version PROJECT_ROOT = Path( __file__ ).resolve().parents[2] DOCUMENTS_FILE = ( PROJECT_ROOT / "data" / "documents.json" ) CHUNKS_FILE = ( PROJECT_ROOT / "data" / "chunks.json" ) CONSTRAINTS = ( ('zpwiki_index_state_unique', 'IndexState', 'id'), ( "zpwiki_document_path_unique", "Document", "path", ), ( "zpwiki_person_id_unique", "Person", "id", ), ( "zpwiki_document_author_id_unique", "DocumentAuthor", "id", ), ( "zpwiki_category_name_unique", "Category", "name", ), ( "zpwiki_topic_id_unique", "Topic", "id", ), ( "zpwiki_keyword_id_unique", "Keyword", "id", ), ( "zpwiki_thesis_id_unique", "Thesis", "id", ), ( "zpwiki_work_id_unique", "Work", "id", ), ) LEGACY_CONSTRAINTS = ( "zpwiki_author_id_unique", ) def graph_version(session) -> dict | None: record = session.run('MATCH (s:IndexState {id:$scope}) RETURN s.metadata_json AS metadata', scope=GRAPH_SCOPE).single() return json.loads(record['metadata']) if record and record['metadata'] else None def write_graph_transaction(tx, payload: dict, *, replace: bool = True, version: dict | None = None) -> None: """All destructive data changes belong to the caller's single transaction.""" if replace: clear_managed_graph(tx) for key, query in NODE_QUERIES.items(): write_rows(tx, query, payload[key]) for key, query in RELATIONSHIP_QUERIES.items(): write_rows(tx, query, payload[key]) if replace: nodes, relationships = graph_counts(tx) labels = {'people': 'Person', 'document_authors': 'DocumentAuthor', 'documents': 'Document', 'categories': 'Category', 'topics': 'Topic', 'keywords': 'Keyword', 'theses': 'Thesis', 'works': 'Work'} types = {'person_documents': 'HAS_DOCUMENT', 'document_author_documents': 'AUTHORED_DOCUMENT', 'document_categories': 'IN_CATEGORY', 'document_keywords': 'HAS_KEYWORD', 'document_describes_topics': 'DESCRIBES', 'person_theses': 'AUTHORED', 'person_works': 'HAS_WORK'} expected_nodes = {labels[key]: len(payload[key]) for key in labels if payload[key]} expected_relations = {types[key]: len(payload[key]) for key in types if payload[key]} evidenced = len(payload['thesis_documents']) + len(payload['work_documents']) if evidenced: expected_relations['EVIDENCED_BY'] = evidenced if dict(nodes) != expected_nodes or dict(relationships) != expected_relations: raise RuntimeError('Neo4j počty nezodpovedajú pripravenému grafu; transakcia sa zruší.') if version is None: tx.run('MATCH (s:IndexState {id:$scope}) DELETE s', scope=GRAPH_SCOPE).consume() else: # Operational state is deliberately outside the knowledge graph scope. tx.run('MERGE (s:IndexState {id:$scope}) SET s.metadata_json=$metadata', scope=GRAPH_SCOPE, metadata=json.dumps(version, sort_keys=True)).consume() NODE_QUERIES = { "people": """ UNWIND $rows AS row MERGE (n:Person {id: row.id}) SET n += row """, "document_authors": """ UNWIND $rows AS row MERGE (n:DocumentAuthor {id: row.id}) SET n += row """, "documents": """ UNWIND $rows AS row MERGE (n:Document {path: row.path}) SET n += row """, "categories": """ UNWIND $rows AS row MERGE (n:Category {name: row.name}) SET n += row """, "topics": """ UNWIND $rows AS row MERGE (n:Topic {id: row.id}) SET n += row """, "keywords": """ UNWIND $rows AS row MERGE (n:Keyword {id: row.id}) SET n += row """, "theses": """ UNWIND $rows AS row MERGE (n:Thesis {id: row.id}) SET n += row """, "works": """ UNWIND $rows AS row MERGE (n:Work {id: row.id}) SET n += row """, } RELATIONSHIP_QUERIES = { "person_documents": """ UNWIND $rows AS row MATCH (a:Person { id: row.person_id }) MATCH (b:Document { path: row.document_path }) MERGE (a)-[:HAS_DOCUMENT]->(b) """, "document_author_documents": """ UNWIND $rows AS row MATCH (a:DocumentAuthor { id: row.document_author_id }) MATCH (b:Document { path: row.document_path }) MERGE (a)-[:AUTHORED_DOCUMENT]->(b) """, "document_categories": """ UNWIND $rows AS row MATCH (a:Document { path: row.document_path }) MATCH (b:Category { name: row.category_name }) MERGE (a)-[:IN_CATEGORY]->(b) """, "document_keywords": """ UNWIND $rows AS row MATCH (a:Document { path: row.document_path }) MATCH (b:Keyword { id: row.keyword_id }) MERGE (a)-[:HAS_KEYWORD]->(b) """, "document_describes_topics": """ UNWIND $rows AS row MATCH (a:Document { path: row.document_path }) MATCH (b:Topic { id: row.topic_id }) MERGE (a)-[:DESCRIBES]->(b) """, "person_theses": """ UNWIND $rows AS row MATCH (a:Person { id: row.person_id }) MATCH (b:Thesis { id: row.thesis_id }) MERGE (a)-[:AUTHORED]->(b) """, "person_works": """ UNWIND $rows AS row MATCH (a:Person { id: row.person_id }) MATCH (b:Work { id: row.work_id }) MERGE (a)-[:HAS_WORK]->(b) """, "thesis_documents": """ UNWIND $rows AS row MATCH (a:Thesis { id: row.thesis_id }) MATCH (b:Document { path: row.document_path }) MERGE (a)-[:EVIDENCED_BY]->(b) """, "work_documents": """ UNWIND $rows AS row MATCH (a:Work { id: row.work_id }) MATCH (b:Document { path: row.document_path }) MERGE (a)-[:EVIDENCED_BY]->(b) """, } def load_json( path: Path, ) -> list[dict[str, Any]]: if not path.exists(): raise FileNotFoundError( f"Missing file: {path}" ) data = json.loads( path.read_text( encoding="utf-8" ) ) if not isinstance( data, list, ): raise ValueError( f"Expected list in {path}" ) return data def create_constraints( session, ) -> None: for ( name, label, property_name, ) in CONSTRAINTS: session.run( ( f"CREATE CONSTRAINT " f"{name} IF NOT EXISTS " f"FOR (n:{label}) " f"REQUIRE n.{property_name} " f"IS UNIQUE" ) ).consume() def drop_legacy_constraints( session, ) -> None: for name in LEGACY_CONSTRAINTS: session.run( ( f"DROP CONSTRAINT " f"{name} IF EXISTS" ) ).consume() def clear_managed_graph( session, ) -> None: session.run( """ MATCH (n) WHERE n.graph_scope = $scope DETACH DELETE n """, scope=GRAPH_SCOPE, ).consume() def write_rows( session, query: str, rows: list[dict[str, Any]], ) -> None: if not rows: return session.run( query, rows=rows, ).consume() def graph_counts( session, ) -> tuple[ list[tuple[str, int]], list[tuple[str, int]], ]: node_rows = session.run( """ MATCH (n) WHERE n.graph_scope = $scope UNWIND labels(n) AS label RETURN label, count(*) AS count ORDER BY label """, scope=GRAPH_SCOPE, ) nodes = [ ( str(record["label"]), int(record["count"]), ) for record in node_rows ] relationship_rows = session.run( """ MATCH (a)-[r]->(b) WHERE a.graph_scope = $scope AND b.graph_scope = $scope RETURN type(r) AS relationship, count(*) AS count ORDER BY relationship """, scope=GRAPH_SCOPE, ) relationships = [ ( str( record[ "relationship" ] ), int(record["count"]), ) for record in relationship_rows ] return nodes, relationships def titled_entity_counts( session, label: str, ) -> tuple[int, int]: if label not in { "Thesis", "Work", }: raise ValueError( f"Unsupported label: {label}" ) record = session.run( f""" MATCH (n:{label}) WHERE n.graph_scope = $scope RETURN count(n) AS total, count(n.title) AS with_title """, scope=GRAPH_SCOPE, ).single() if record is None: return 0, 0 return ( int(record["total"]), int(record["with_title"]), ) def print_payload_summary( payload: dict[ str, list[dict[str, Any]], ], ) -> None: print() print("Prepared GraphRAG payload") print("=" * 60) for key in ( "people", "document_authors", "documents", "categories", "topics", "keywords", "theses", "works", ): print( f"{key:<24}" f"{len(payload[key]):>6}" ) print() print("Relationships") for key in ( "person_documents", "document_author_documents", "document_categories", "document_keywords", "document_describes_topics", "person_theses", "person_works", "thesis_documents", "work_documents", ): print( f"{key:<30}" f"{len(payload[key]):>6}" ) print("=" * 60) def print_database_summary( session, ) -> None: nodes, relationships = ( graph_counts( session ) ) print() print("Neo4j GraphRAG knowledge graph") print("=" * 60) print("Nodes") for label, count in nodes: print( f" {label:<22}" f"{count:>6}" ) print() print("Relationships") for relationship, count in relationships: print( f" {relationship:<22}" f"{count:>6}" ) thesis_total, thesis_with_title = ( titled_entity_counts( session, "Thesis", ) ) work_total, work_with_title = ( titled_entity_counts( session, "Work", ) ) print() print( "Theses with extracted title: " f"{thesis_with_title}/" f"{thesis_total}" ) print( "Works with extracted title: " f"{work_with_title}/" f"{work_total}" ) print("=" * 60) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "--replace", action=argparse.BooleanOptionalAction, default=True, ) args = parser.parse_args() settings = get_neo4j_settings() with create_neo4j_driver( settings ) as driver: driver.verify_connectivity() with driver.session( database=settings.database ) as session: drop_legacy_constraints( session ) create_constraints( session ) import hashlib with publication_lock(DATA_DIR, exclusive=True): if (DATA_DIR / PENDING_FILE).exists(): raise RuntimeError('Nedokončená spoločná synchronizácia; použite tools.rag.rebuild_index.') payload = build_graph_payload(load_json(DOCUMENTS_FILE), load_json(CHUNKS_FILE)) print_payload_summary(payload) version = read_version(DATA_DIR) if not args.replace: version = None # An incremental merge cannot certify removed source records. if version and any(hashlib.sha256(path.read_bytes()).hexdigest() != version[key] for path, key in [(DOCUMENTS_FILE, 'documents_sha256'), (CHUNKS_FILE, 'chunks_sha256')]): version = None # A standalone rebuild must not claim stale provenance. with session.begin_transaction() as tx: write_graph_transaction(tx, payload, replace=args.replace, version=version) tx.commit() print_database_summary( session ) if __name__ == "__main__": main()