181 lines
8.2 KiB
Python
181 lines
8.2 KiB
Python
"""Neo4j projection of read CRZP records and fixed relation templates.
|
|
|
|
Labels are separate from the ZP Wiki graph (CrzpThesis, CrzpPerson,
|
|
CrzpKeyword, CrzpSchool), so the two sources never mix. People are keyed by
|
|
the name exactly as CRZP writes it; namesakes are not merged with ZP Wiki
|
|
people. Only these fixed templates run; the model never writes Cypher. When
|
|
Neo4j is unavailable, the same aggregation runs over the read rows in Python.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import threading
|
|
from collections import Counter
|
|
|
|
from tools.crzp.source.parse import fold
|
|
|
|
logger = logging.getLogger(__name__)
|
|
ROLE_REL = {"author": "AUTHOR", "supervisor": "SUPERVISOR", "opponent": "OPPONENT"}
|
|
ROLE_FIELD = {"author": "authors", "supervisor": "supervisors", "opponent": "opponents"}
|
|
_driver = None
|
|
_lock = threading.Lock()
|
|
|
|
PROJECT = """
|
|
UNWIND $rows AS row
|
|
MERGE (t:CrzpThesis {sid: row.sid})
|
|
SET t.title = row.title, t.year = row.year, t.work_type = row.work_type, t.url = row.url
|
|
MERGE (s:CrzpSchool {name: row.school})
|
|
MERGE (t)-[:AT]->(s)
|
|
FOREACH (name IN row.authors | MERGE (p:CrzpPerson {name: name}) MERGE (t)-[:AUTHOR]->(p))
|
|
FOREACH (name IN row.supervisors | MERGE (p:CrzpPerson {name: name}) MERGE (t)-[:SUPERVISOR]->(p))
|
|
FOREACH (name IN row.opponents | MERGE (p:CrzpPerson {name: name}) MERGE (t)-[:OPPONENT]->(p))
|
|
FOREACH (k IN row.keywords | MERGE (w:CrzpKeyword {key: k.key}) ON CREATE SET w.label = k.label
|
|
MERGE (t)-[:HAS_KEYWORD]->(w))
|
|
"""
|
|
# Relation templates: {source} and {target} are taken only from ROLE_REL.
|
|
RELATION = """
|
|
MATCH (t:CrzpThesis)-[:{source}]->(p:CrzpPerson) WHERE t.sid IN $sids AND p.name IN $names
|
|
MATCH (t)-[:{target}]->(o:CrzpPerson) WHERE NOT o.name IN $names
|
|
RETURN o.name AS name, count(DISTINCT t) AS works ORDER BY works DESC, name
|
|
"""
|
|
SHARED_KEYWORDS = """
|
|
MATCH (t:CrzpThesis {sid: $sid})-[:HAS_KEYWORD]->(k:CrzpKeyword)<-[:HAS_KEYWORD]-(o:CrzpThesis)
|
|
WHERE o.sid IN $candidates
|
|
RETURN o.sid AS sid, collect(k.label) AS shared ORDER BY size(shared) DESC, sid
|
|
"""
|
|
|
|
|
|
# Links to the ZP Wiki graph, written by code only (never by the model):
|
|
# a CRZP person with a work at TUKE whose name equals a ZP Wiki person (probably the same person; CRZP
|
|
# has no identifiers), and a CRZP keyword that is a ZP Wiki topic or keyword (same words or reviewed synonyms).
|
|
LINK_PEOPLE = """
|
|
UNWIND $pairs AS pair
|
|
MATCH (c:CrzpPerson {name: pair.crzp})
|
|
MATCH (p:Person {name: pair.wiki})
|
|
MERGE (c)-[r:PROBABLY_SAME]->(p) SET r.basis = 'rovnaké meno, práca na TUKE'
|
|
"""
|
|
LINK_TOPICS = """
|
|
UNWIND $pairs AS pair
|
|
MATCH (k:CrzpKeyword {key: pair.key})
|
|
MATCH (t) WHERE (t:Topic OR t:Keyword) AND t.name = pair.wiki
|
|
MERGE (k)-[:SAME_TOPIC]->(t)
|
|
"""
|
|
WIKI_NAMES = "MATCH (p:Person) RETURN p.name AS name"
|
|
WIKI_TOPICS = "MATCH (t) WHERE t:Topic OR t:Keyword RETURN DISTINCT t.name AS name"
|
|
PERSON_WORKS = """
|
|
MATCH (p:Person {name: $name})<-[:PROBABLY_SAME]-(c:CrzpPerson)<-[r:AUTHOR|SUPERVISOR|OPPONENT]-(t:CrzpThesis)
|
|
RETURN t.sid AS sid, t.title AS title, t.year AS year, t.work_type AS work_type, type(r) AS role ORDER BY t.year DESC
|
|
"""
|
|
_wiki = {"at": 0.0, "names": [], "topics": []}
|
|
|
|
|
|
def link_pairs(rows: list[dict], wiki_names: list[str], wiki_topics: list[str]) -> tuple[list[dict], list[dict]]:
|
|
"""(person pairs, keyword pairs) for the rows; pure, so it is tested without Neo4j."""
|
|
from tools.crzp.source.parse import display_name
|
|
from tools.retrieval.synonyms import base_name
|
|
names = {fold(n): n for n in wiki_names if n}
|
|
topics = {}
|
|
for topic in wiki_topics:
|
|
if topic:
|
|
topics.setdefault(fold(topic), topic)
|
|
topics.setdefault(fold(base_name(topic)), topic)
|
|
people, keywords = {}, {}
|
|
for row in rows:
|
|
if "technicka univerzita v kosiciach" not in fold(row.get("school") or ""):
|
|
continue
|
|
for field in ("authors", "supervisors", "opponents"):
|
|
for name in row.get(field, []):
|
|
wiki = names.get(fold(display_name(name)))
|
|
if wiki:
|
|
people[name] = wiki
|
|
for row in rows:
|
|
for keyword in row.get("keywords", []):
|
|
wiki = topics.get(fold(keyword)) or topics.get(fold(base_name(keyword)))
|
|
if wiki:
|
|
keywords[fold(keyword)] = wiki
|
|
return ([{"crzp": c, "wiki": w} for c, w in people.items()], [{"key": k, "wiki": w} for k, w in keywords.items()])
|
|
|
|
|
|
def link(session, rows: list[dict]) -> None:
|
|
import time
|
|
if time.time() - _wiki["at"] > 600:
|
|
_wiki.update(at=time.time(), names=session.run(WIKI_NAMES).value(), topics=session.run(WIKI_TOPICS).value())
|
|
people, keywords = link_pairs(rows, _wiki["names"], _wiki["topics"])
|
|
if people:
|
|
session.execute_write(lambda tx: tx.run(LINK_PEOPLE, pairs=people).consume())
|
|
if keywords:
|
|
session.execute_write(lambda tx: tx.run(LINK_TOPICS, pairs=keywords).consume())
|
|
|
|
|
|
def person_works(name: str) -> list[dict]:
|
|
"""CRZP works linked to a ZP Wiki person in the graph ([] without Neo4j)."""
|
|
try:
|
|
with _session() as session:
|
|
return [dict(r) for r in session.run(PERSON_WORKS, name=name)]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _session():
|
|
global _driver
|
|
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
|
with _lock:
|
|
if _driver is None:
|
|
_driver = create_neo4j_driver(connection_timeout=3, max_transaction_retry_time=3)
|
|
return _driver.session(database=get_neo4j_settings().database)
|
|
|
|
|
|
def project(rows: list[dict]) -> bool:
|
|
"""Write the rows (idempotent MERGE); False when Neo4j is unavailable."""
|
|
data = [{**{k: row.get(k) for k in ("sid", "title", "year", "work_type", "url")},
|
|
"school": row.get("school") or "", "authors": row.get("authors", []),
|
|
"supervisors": row.get("supervisors", []), "opponents": row.get("opponents", []),
|
|
"keywords": [{"key": fold(k), "label": k} for k in row.get("keywords", [])]}
|
|
for row in rows if row.get("sid")]
|
|
if not data:
|
|
return False
|
|
try:
|
|
with _session() as session:
|
|
session.execute_write(lambda tx: tx.run(PROJECT, rows=data).consume())
|
|
try:
|
|
link(session, rows)
|
|
except Exception:
|
|
logger.warning("Linking CRZP records to the ZP Wiki graph failed.")
|
|
return True
|
|
except Exception:
|
|
logger.warning("CRZP projection to Neo4j failed; Python aggregation is used.")
|
|
return False
|
|
|
|
|
|
def relation(rows: list[dict], names: list[str], source: str, target: str) -> list[tuple[str, int]]:
|
|
"""People in the target role on the works where `names` have the source role."""
|
|
sids = [r["sid"] for r in rows if r.get("sid")]
|
|
if len(sids) == len(rows) and project(rows):
|
|
try:
|
|
with _session() as session:
|
|
result = session.run(RELATION.format(source=ROLE_REL[source], target=ROLE_REL[target]),
|
|
sids=sids, names=names)
|
|
return [(r["name"], r["works"]) for r in result]
|
|
except Exception:
|
|
logger.warning("CRZP relation query failed; Python aggregation is used.")
|
|
counts = Counter()
|
|
for row in rows:
|
|
if any(n in names for n in row.get(ROLE_FIELD[source], [])):
|
|
counts.update({o for o in row.get(ROLE_FIELD[target], []) if o not in names})
|
|
return sorted(counts.items(), key=lambda item: (-item[1], item[0]))
|
|
|
|
|
|
def shared_keywords(work: dict, candidates: list[dict]) -> list[tuple[dict, list[str]]]:
|
|
"""Candidates sharing CRZP keywords with the work, most shared first."""
|
|
by_sid = {c["sid"]: c for c in candidates if c.get("sid") and c.get("sid") != work.get("sid")}
|
|
if work.get("sid") and project([work, *by_sid.values()]):
|
|
try:
|
|
with _session() as session:
|
|
result = session.run(SHARED_KEYWORDS, sid=work["sid"], candidates=list(by_sid))
|
|
return [(by_sid[r["sid"]], r["shared"]) for r in result if r["sid"] in by_sid]
|
|
except Exception:
|
|
logger.warning("CRZP keyword query failed; Python comparison is used.")
|
|
own = {fold(k): k for k in work.get("keywords", [])}
|
|
scored = [(c, [own[fold(k)] for k in c.get("keywords", []) if fold(k) in own]) for c in by_sid.values()]
|
|
return sorted([s for s in scored if s[1]], key=lambda s: (-len(s[1]), s[0]["sid"]))
|