2667 lines
66 KiB
Python
2667 lines
66 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
from typing import Any, Iterable
|
|
|
|
|
|
GRAPH_SCOPE = "zpwiki"
|
|
DEFAULT_LIMIT = 40
|
|
DEFAULT_MAX_ENTITIES = 8
|
|
|
|
RELATIONSHIP_TYPES = (
|
|
"HAS_DOCUMENT|AUTHORED|IN_CATEGORY|HAS_KEYWORD|"
|
|
"DESCRIBES|HAS_WORK|EVIDENCED_BY|AUTHORED_DOCUMENT"
|
|
)
|
|
|
|
CATALOG_QUERY = """
|
|
MATCH (n)
|
|
WHERE n.graph_scope = $scope
|
|
AND (
|
|
n:Person
|
|
OR n:Thesis
|
|
OR n:Work
|
|
OR n:Keyword
|
|
OR n:Topic
|
|
OR n:Category
|
|
OR n:DocumentAuthor
|
|
)
|
|
RETURN
|
|
elementId(n) AS element_id,
|
|
CASE
|
|
WHEN n:Person THEN 'Person'
|
|
WHEN n:Thesis THEN 'Thesis'
|
|
WHEN n:Work THEN 'Work'
|
|
WHEN n:Keyword THEN 'Keyword'
|
|
WHEN n:Topic THEN 'Topic'
|
|
WHEN n:Category THEN 'Category'
|
|
WHEN n:DocumentAuthor THEN 'DocumentAuthor'
|
|
ELSE 'Other'
|
|
END AS kind,
|
|
n.id AS id,
|
|
n.name AS name,
|
|
n.title AS title,
|
|
n.heading AS heading,
|
|
n.start_year AS start_year,
|
|
n.year AS year,
|
|
n.thesis_type AS thesis_type,
|
|
n.thesis_type_label AS thesis_type_label,
|
|
n.work_type AS work_type,
|
|
n.work_type_label AS work_type_label,
|
|
n.person_kind AS person_kind,
|
|
n.source_document AS source_document
|
|
ORDER BY kind, name, title, heading
|
|
"""
|
|
|
|
NEIGHBORHOOD_QUERY = f"""
|
|
MATCH (start)
|
|
WHERE elementId(start) IN $start_ids
|
|
MATCH p=(start)-[:{RELATIONSHIP_TYPES}*1..2]-(target)
|
|
WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)
|
|
AND (
|
|
size($target_kinds) = 0
|
|
OR any(
|
|
node IN nodes(p)
|
|
WHERE
|
|
('Person' IN $target_kinds AND node:Person)
|
|
OR ('Thesis' IN $target_kinds AND node:Thesis)
|
|
OR ('Work' IN $target_kinds AND node:Work)
|
|
OR ('Keyword' IN $target_kinds AND node:Keyword)
|
|
OR ('Topic' IN $target_kinds AND node:Topic)
|
|
OR ('Category' IN $target_kinds AND node:Category)
|
|
OR (
|
|
'DocumentAuthor' IN $target_kinds
|
|
AND node:DocumentAuthor
|
|
)
|
|
OR ('Document' IN $target_kinds AND node:Document)
|
|
)
|
|
)
|
|
RETURN
|
|
[node IN nodes(p) | {{
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Thesis THEN 'Thesis'
|
|
WHEN node:Work THEN 'Work'
|
|
WHEN node:Document THEN 'Document'
|
|
WHEN node:Keyword THEN 'Keyword'
|
|
WHEN node:Topic THEN 'Topic'
|
|
WHEN node:Category THEN 'Category'
|
|
WHEN node:DocumentAuthor THEN 'DocumentAuthor'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
thesis_type: node.thesis_type,
|
|
thesis_type_label: node.thesis_type_label,
|
|
work_type: node.work_type,
|
|
work_type_label: node.work_type_label,
|
|
person_kind: node.person_kind,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY hops
|
|
LIMIT $limit
|
|
"""
|
|
|
|
PAIR_PATH_QUERY = f"""
|
|
MATCH (left), (right)
|
|
WHERE elementId(left) IN $start_ids
|
|
AND elementId(right) IN $start_ids
|
|
AND elementId(left) < elementId(right)
|
|
MATCH p=allShortestPaths(
|
|
(left)-[:{RELATIONSHIP_TYPES}*1..4]-(right)
|
|
)
|
|
WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)
|
|
RETURN
|
|
[node IN nodes(p) | {{
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Thesis THEN 'Thesis'
|
|
WHEN node:Work THEN 'Work'
|
|
WHEN node:Document THEN 'Document'
|
|
WHEN node:Keyword THEN 'Keyword'
|
|
WHEN node:Topic THEN 'Topic'
|
|
WHEN node:Category THEN 'Category'
|
|
WHEN node:DocumentAuthor THEN 'DocumentAuthor'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
thesis_type: node.thesis_type,
|
|
thesis_type_label: node.thesis_type_label,
|
|
work_type: node.work_type,
|
|
work_type_label: node.work_type_label,
|
|
person_kind: node.person_kind,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY hops
|
|
LIMIT $limit
|
|
"""
|
|
|
|
THESIS_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:AUTHORED]->
|
|
(thesis:Thesis)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND thesis.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($thesis_ids) = 0
|
|
OR elementId(thesis) IN $thesis_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR thesis.year IN $years
|
|
)
|
|
AND (
|
|
size($thesis_types) = 0
|
|
OR thesis.thesis_type IN $thesis_types
|
|
)
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
person.start_year AS start_year,
|
|
elementId(thesis) AS thesis_element_id,
|
|
thesis.id AS thesis_id,
|
|
thesis.title AS title,
|
|
thesis.heading AS heading,
|
|
thesis.year AS year,
|
|
thesis.thesis_type AS thesis_type,
|
|
thesis.thesis_type_label AS thesis_type_label,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url,
|
|
thesis.source_chunk_ids AS source_chunk_ids
|
|
ORDER BY person, year, thesis_type, title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
THESIS_PATHS_QUERY = """
|
|
MATCH p=
|
|
(person:Person)
|
|
-[:AUTHORED]->
|
|
(thesis:Thesis)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND thesis.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($thesis_ids) = 0
|
|
OR elementId(thesis) IN $thesis_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR thesis.year IN $years
|
|
)
|
|
AND (
|
|
size($thesis_types) = 0
|
|
OR thesis.thesis_type IN $thesis_types
|
|
)
|
|
RETURN
|
|
[node IN nodes(p) | {
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Thesis THEN 'Thesis'
|
|
WHEN node:Document THEN 'Document'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
thesis_type: node.thesis_type,
|
|
thesis_type_label: node.thesis_type_label,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY
|
|
person.name,
|
|
thesis.year,
|
|
thesis.thesis_type,
|
|
thesis.title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
WORK_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:HAS_WORK]->
|
|
(work:Work)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND work.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($work_ids) = 0
|
|
OR elementId(work) IN $work_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR work.year IN $years
|
|
)
|
|
AND (
|
|
size($work_types) = 0
|
|
OR work.work_type IN $work_types
|
|
)
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
person.start_year AS start_year,
|
|
elementId(work) AS work_element_id,
|
|
work.id AS work_id,
|
|
work.title AS title,
|
|
work.heading AS heading,
|
|
work.year AS year,
|
|
work.work_type AS work_type,
|
|
work.work_type_label AS work_type_label,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url,
|
|
work.source_chunk_ids AS source_chunk_ids
|
|
ORDER BY person, year, work_type, title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
WORK_PATHS_QUERY = """
|
|
MATCH p=
|
|
(person:Person)
|
|
-[:HAS_WORK]->
|
|
(work:Work)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND work.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($work_ids) = 0
|
|
OR elementId(work) IN $work_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR work.year IN $years
|
|
)
|
|
AND (
|
|
size($work_types) = 0
|
|
OR work.work_type IN $work_types
|
|
)
|
|
RETURN
|
|
[node IN nodes(p) | {
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Work THEN 'Work'
|
|
WHEN node:Document THEN 'Document'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
work_type: node.work_type,
|
|
work_type_label: node.work_type_label,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY
|
|
person.name,
|
|
work.year,
|
|
work.work_type,
|
|
work.title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
START_YEAR_FACTS_QUERY = """
|
|
MATCH (person:Person)
|
|
WHERE person.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR person.start_year IN $years
|
|
)
|
|
OPTIONAL MATCH (document:Document)
|
|
WHERE document.graph_scope = $scope
|
|
AND document.path = person.source_document
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
person.start_year AS start_year,
|
|
person.source_document AS document_path,
|
|
document.source_url AS source_url
|
|
ORDER BY person
|
|
LIMIT $limit
|
|
"""
|
|
|
|
PERSON_KEYWORD_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(document:Document)
|
|
-[:HAS_KEYWORD]->
|
|
(keyword:Keyword)
|
|
WHERE person.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND keyword.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($keyword_ids) = 0
|
|
OR elementId(keyword) IN $keyword_ids
|
|
)
|
|
WITH
|
|
person,
|
|
keyword,
|
|
collect(DISTINCT document) AS documents
|
|
WITH
|
|
person,
|
|
keyword,
|
|
head(
|
|
[
|
|
document IN documents
|
|
WHERE document.path = person.source_document
|
|
]
|
|
+ documents
|
|
) AS document
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
elementId(keyword) AS keyword_element_id,
|
|
keyword.id AS keyword_id,
|
|
keyword.name AS keyword,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url
|
|
ORDER BY person, keyword
|
|
LIMIT $limit
|
|
"""
|
|
|
|
PERSON_CATEGORY_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(document:Document)
|
|
-[:IN_CATEGORY]->
|
|
(category:Category)
|
|
WHERE person.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND category.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($category_ids) = 0
|
|
OR elementId(category) IN $category_ids
|
|
)
|
|
WITH
|
|
person,
|
|
category,
|
|
collect(DISTINCT document) AS documents
|
|
WITH
|
|
person,
|
|
category,
|
|
head(
|
|
[
|
|
document IN documents
|
|
WHERE document.path = person.source_document
|
|
]
|
|
+ documents
|
|
) AS document
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
elementId(category) AS category_element_id,
|
|
category.name AS category,
|
|
category.category_type AS category_type,
|
|
category.year AS category_year,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url
|
|
ORDER BY person, category
|
|
LIMIT $limit
|
|
"""
|
|
|
|
SHARED_KEYWORD_FACTS_QUERY = """
|
|
MATCH
|
|
(left:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(left_document:Document)
|
|
-[:HAS_KEYWORD]->
|
|
(keyword:Keyword)
|
|
MATCH
|
|
(right:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(right_document:Document)
|
|
-[:HAS_KEYWORD]->
|
|
(keyword)
|
|
WHERE left.graph_scope = $scope
|
|
AND right.graph_scope = $scope
|
|
AND left_document.graph_scope = $scope
|
|
AND right_document.graph_scope = $scope
|
|
AND keyword.graph_scope = $scope
|
|
AND elementId(left) IN $person_ids
|
|
AND elementId(right) IN $person_ids
|
|
AND elementId(left) < elementId(right)
|
|
WITH
|
|
left,
|
|
right,
|
|
keyword,
|
|
collect(DISTINCT left_document) AS left_documents,
|
|
collect(DISTINCT right_document) AS right_documents
|
|
WITH
|
|
left,
|
|
right,
|
|
keyword,
|
|
head(
|
|
[
|
|
document IN left_documents
|
|
WHERE document.path = left.source_document
|
|
]
|
|
+ left_documents
|
|
) AS left_document,
|
|
head(
|
|
[
|
|
document IN right_documents
|
|
WHERE document.path = right.source_document
|
|
]
|
|
+ right_documents
|
|
) AS right_document
|
|
RETURN
|
|
elementId(left) AS left_person_element_id,
|
|
left.name AS left_person,
|
|
left_document.path AS left_document_path,
|
|
left_document.source_url AS left_source_url,
|
|
elementId(right) AS right_person_element_id,
|
|
right.name AS right_person,
|
|
right_document.path AS right_document_path,
|
|
right_document.source_url AS right_source_url,
|
|
elementId(keyword) AS keyword_element_id,
|
|
keyword.id AS keyword_id,
|
|
keyword.name AS keyword
|
|
ORDER BY left_person, right_person, keyword
|
|
LIMIT $limit
|
|
"""
|
|
|
|
SHARED_CATEGORY_FACTS_QUERY = """
|
|
MATCH
|
|
(left:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(left_document:Document)
|
|
-[:IN_CATEGORY]->
|
|
(category:Category)
|
|
MATCH
|
|
(right:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(right_document:Document)
|
|
-[:IN_CATEGORY]->
|
|
(category)
|
|
WHERE left.graph_scope = $scope
|
|
AND right.graph_scope = $scope
|
|
AND left_document.graph_scope = $scope
|
|
AND right_document.graph_scope = $scope
|
|
AND category.graph_scope = $scope
|
|
AND elementId(left) IN $person_ids
|
|
AND elementId(right) IN $person_ids
|
|
AND elementId(left) < elementId(right)
|
|
WITH
|
|
left,
|
|
right,
|
|
category,
|
|
collect(DISTINCT left_document) AS left_documents,
|
|
collect(DISTINCT right_document) AS right_documents
|
|
WITH
|
|
left,
|
|
right,
|
|
category,
|
|
head(
|
|
[
|
|
document IN left_documents
|
|
WHERE document.path = left.source_document
|
|
]
|
|
+ left_documents
|
|
) AS left_document,
|
|
head(
|
|
[
|
|
document IN right_documents
|
|
WHERE document.path = right.source_document
|
|
]
|
|
+ right_documents
|
|
) AS right_document
|
|
RETURN
|
|
elementId(left) AS left_person_element_id,
|
|
left.name AS left_person,
|
|
left_document.path AS left_document_path,
|
|
left_document.source_url AS left_source_url,
|
|
elementId(right) AS right_person_element_id,
|
|
right.name AS right_person,
|
|
right_document.path AS right_document_path,
|
|
right_document.source_url AS right_source_url,
|
|
elementId(category) AS category_element_id,
|
|
category.name AS category,
|
|
category.category_type AS category_type,
|
|
category.year AS category_year
|
|
ORDER BY left_person, right_person, category
|
|
LIMIT $limit
|
|
"""
|
|
|
|
SHARED_DOCUMENT_AUTHOR_FACTS_QUERY = """
|
|
MATCH
|
|
(author:DocumentAuthor)
|
|
-[:AUTHORED_DOCUMENT]->
|
|
(left_document:Document)
|
|
<-[:HAS_DOCUMENT]-
|
|
(left:Person)
|
|
MATCH
|
|
(author)
|
|
-[:AUTHORED_DOCUMENT]->
|
|
(right_document:Document)
|
|
<-[:HAS_DOCUMENT]-
|
|
(right:Person)
|
|
WHERE author.graph_scope = $scope
|
|
AND left.graph_scope = $scope
|
|
AND right.graph_scope = $scope
|
|
AND left_document.graph_scope = $scope
|
|
AND right_document.graph_scope = $scope
|
|
AND elementId(left) IN $person_ids
|
|
AND elementId(right) IN $person_ids
|
|
AND elementId(left) < elementId(right)
|
|
WITH
|
|
left,
|
|
right,
|
|
author,
|
|
collect(DISTINCT left_document) AS left_documents,
|
|
collect(DISTINCT right_document) AS right_documents
|
|
WITH
|
|
left,
|
|
right,
|
|
author,
|
|
head(
|
|
[
|
|
document IN left_documents
|
|
WHERE document.path = left.source_document
|
|
]
|
|
+ left_documents
|
|
) AS left_document,
|
|
head(
|
|
[
|
|
document IN right_documents
|
|
WHERE document.path = right.source_document
|
|
]
|
|
+ right_documents
|
|
) AS right_document
|
|
RETURN
|
|
elementId(left) AS left_person_element_id,
|
|
left.name AS left_person,
|
|
left_document.path AS left_document_path,
|
|
left_document.source_url AS left_source_url,
|
|
elementId(right) AS right_person_element_id,
|
|
right.name AS right_person,
|
|
right_document.path AS right_document_path,
|
|
right_document.source_url AS right_source_url,
|
|
elementId(author) AS document_author_element_id,
|
|
author.id AS document_author_id,
|
|
author.name AS document_author
|
|
ORDER BY left_person, right_person, document_author
|
|
LIMIT $limit
|
|
"""
|
|
|
|
|
|
def normalize_for_match(
|
|
value: str,
|
|
) -> str:
|
|
decomposed = unicodedata.normalize(
|
|
"NFKD",
|
|
str(value),
|
|
)
|
|
|
|
without_marks = "".join(
|
|
character
|
|
for character in decomposed
|
|
if not unicodedata.combining(
|
|
character
|
|
)
|
|
)
|
|
|
|
normalized = re.sub(
|
|
r"[^a-zA-Z0-9]+",
|
|
" ",
|
|
without_marks.casefold(),
|
|
)
|
|
|
|
return re.sub(
|
|
r"\s+",
|
|
" ",
|
|
normalized,
|
|
).strip()
|
|
|
|
|
|
def normalized_tokens(
|
|
value: str,
|
|
) -> list[str]:
|
|
normalized = normalize_for_match(
|
|
value
|
|
)
|
|
|
|
if not normalized:
|
|
return []
|
|
|
|
return normalized.split()
|
|
|
|
|
|
def has_stem(
|
|
tokens: Iterable[str],
|
|
stem: str,
|
|
) -> bool:
|
|
return any(
|
|
token.startswith(
|
|
stem
|
|
)
|
|
for token in tokens
|
|
)
|
|
|
|
|
|
def explicit_keyword_phrase(query: str) -> str | None:
|
|
"""Identify a requested tag without treating 'ZP Wiki' as the tag ZP."""
|
|
match = re.search(
|
|
r"\b(?:klucove\s+slovo|klucovym\s+slovom|klucoveho\s+slova|"
|
|
r"tagom|tagu|tag)\s+(.+)",
|
|
normalize_for_match(query),
|
|
)
|
|
if not match:
|
|
return None
|
|
phrase = re.split(
|
|
r"\s+(?:a\s+(?:jej|jeho|ich|rok|pomenuj|uved\w*)|ak\s+|uved\w*\s+|"
|
|
r"pomenuj\s+|maju\s+|ma\s+)",
|
|
match.group(1),
|
|
maxsplit=1,
|
|
)[0].strip()
|
|
return phrase or None
|
|
|
|
|
|
def extract_query_constraints(
|
|
query: str,
|
|
) -> dict[str, Any]:
|
|
normalized = normalize_for_match(
|
|
query
|
|
)
|
|
|
|
tokens = normalized.split()
|
|
token_set = set(
|
|
tokens
|
|
)
|
|
|
|
years = sorted(
|
|
{
|
|
int(value)
|
|
for value in re.findall(
|
|
r"\b(20\d{2})\b",
|
|
normalized,
|
|
)
|
|
}
|
|
)
|
|
|
|
has_work_word = has_stem(
|
|
tokens,
|
|
"prac",
|
|
)
|
|
|
|
has_project_word = has_stem(
|
|
tokens,
|
|
"projekt",
|
|
)
|
|
|
|
thesis_types: list[str] = []
|
|
work_types: list[str] = []
|
|
|
|
if (
|
|
(
|
|
has_stem(
|
|
tokens,
|
|
"bakalar",
|
|
)
|
|
and has_work_word
|
|
)
|
|
or "bp" in token_set
|
|
):
|
|
thesis_types.append(
|
|
"bachelor_thesis"
|
|
)
|
|
|
|
if (
|
|
(
|
|
has_stem(
|
|
tokens,
|
|
"diplom",
|
|
)
|
|
and has_work_word
|
|
)
|
|
or "dp" in token_set
|
|
):
|
|
thesis_types.append(
|
|
"diploma_thesis"
|
|
)
|
|
|
|
if (
|
|
(
|
|
has_stem(
|
|
tokens,
|
|
"dizert",
|
|
)
|
|
and has_work_word
|
|
)
|
|
or "phd" in token_set
|
|
):
|
|
thesis_types.append(
|
|
"doctoral_thesis"
|
|
)
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"diplom",
|
|
)
|
|
and has_project_word
|
|
):
|
|
work_types.append(
|
|
"diploma_project"
|
|
)
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"timov",
|
|
)
|
|
and has_project_word
|
|
):
|
|
work_types.append(
|
|
"team_project"
|
|
)
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"vyskum",
|
|
)
|
|
and has_project_word
|
|
):
|
|
work_types.append(
|
|
"research_project"
|
|
)
|
|
|
|
if "tp" in token_set:
|
|
work_types.append(
|
|
"team_project"
|
|
)
|
|
|
|
if "vp" in token_set:
|
|
work_types.append(
|
|
"research_project"
|
|
)
|
|
|
|
thesis_types = list(
|
|
dict.fromkeys(
|
|
thesis_types
|
|
)
|
|
)
|
|
|
|
work_types = list(
|
|
dict.fromkeys(
|
|
work_types
|
|
)
|
|
)
|
|
|
|
mentions_start_year = (
|
|
has_stem(
|
|
tokens,
|
|
"zaciat",
|
|
)
|
|
and has_stem(
|
|
tokens,
|
|
"stud",
|
|
)
|
|
)
|
|
|
|
mentions_keyword = (
|
|
has_stem(
|
|
tokens,
|
|
"kluc",
|
|
)
|
|
or "tag" in token_set
|
|
or "tagy" in token_set
|
|
)
|
|
|
|
mentions_topic = (
|
|
has_stem(
|
|
tokens,
|
|
"tem",
|
|
)
|
|
)
|
|
|
|
mentions_category = (
|
|
has_stem(
|
|
tokens,
|
|
"kategori",
|
|
)
|
|
)
|
|
|
|
mentions_document_author = (
|
|
has_stem(
|
|
tokens,
|
|
"autor",
|
|
)
|
|
and (
|
|
has_stem(
|
|
tokens,
|
|
"dokument",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"strank",
|
|
)
|
|
or "wiki" in token_set
|
|
)
|
|
)
|
|
|
|
mentions_title = (
|
|
has_stem(tokens, "nazov") or has_stem(tokens, "nazv")
|
|
or (has_work_word and "vola" in token_set)
|
|
)
|
|
mentions_academic_year = (
|
|
has_work_word
|
|
and bool(token_set & {"rok", "roka", "roku", "rokom", "roky", "rokoch"})
|
|
and not mentions_start_year
|
|
)
|
|
|
|
mentions_shared = (
|
|
has_stem(
|
|
tokens,
|
|
"spoloc",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"rovnak",
|
|
)
|
|
or (mentions_academic_year and has_stem(tokens, "spaj"))
|
|
)
|
|
|
|
mentions_person_answer = (
|
|
"kto" in token_set
|
|
or (
|
|
has_stem(
|
|
tokens,
|
|
"ktor",
|
|
)
|
|
and (
|
|
has_stem(
|
|
tokens,
|
|
"student",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"osob",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"clov",
|
|
)
|
|
)
|
|
)
|
|
)
|
|
|
|
mentions_thesis = (
|
|
bool(
|
|
thesis_types
|
|
)
|
|
or (
|
|
has_stem(
|
|
tokens,
|
|
"zaverec",
|
|
)
|
|
and has_work_word
|
|
)
|
|
)
|
|
|
|
mentions_work = bool(
|
|
work_types
|
|
)
|
|
|
|
target_kinds: list[str] = []
|
|
|
|
if mentions_keyword:
|
|
target_kinds.append(
|
|
"Keyword"
|
|
)
|
|
|
|
if mentions_topic:
|
|
target_kinds.extend(
|
|
[
|
|
"Keyword",
|
|
"Topic",
|
|
]
|
|
)
|
|
|
|
if mentions_category:
|
|
target_kinds.append(
|
|
"Category"
|
|
)
|
|
|
|
if mentions_document_author:
|
|
target_kinds.append(
|
|
"DocumentAuthor"
|
|
)
|
|
|
|
if mentions_thesis:
|
|
target_kinds.append(
|
|
"Thesis"
|
|
)
|
|
|
|
if mentions_work:
|
|
target_kinds.append(
|
|
"Work"
|
|
)
|
|
|
|
if (
|
|
mentions_title
|
|
and not mentions_thesis
|
|
and not mentions_work
|
|
):
|
|
target_kinds.extend(
|
|
[
|
|
"Thesis",
|
|
"Work",
|
|
]
|
|
)
|
|
|
|
if (
|
|
years
|
|
and not mentions_start_year
|
|
and not mentions_thesis
|
|
and not mentions_work
|
|
):
|
|
target_kinds.extend(
|
|
[
|
|
"Thesis",
|
|
"Work",
|
|
]
|
|
)
|
|
|
|
if (
|
|
mentions_person_answer
|
|
and not mentions_shared
|
|
):
|
|
target_kinds = [
|
|
"Person"
|
|
]
|
|
|
|
return {
|
|
"years": years,
|
|
"thesis_types": (
|
|
thesis_types
|
|
),
|
|
"work_types": (
|
|
work_types
|
|
),
|
|
"mentions_start_year": (
|
|
mentions_start_year
|
|
),
|
|
"mentions_keyword": (
|
|
mentions_keyword
|
|
),
|
|
"mentions_topic": (
|
|
mentions_topic
|
|
),
|
|
"mentions_category": (
|
|
mentions_category
|
|
),
|
|
"mentions_document_author": (
|
|
mentions_document_author
|
|
),
|
|
"mentions_title": (
|
|
mentions_title
|
|
),
|
|
"mentions_academic_year": mentions_academic_year,
|
|
"keyword_phrase": explicit_keyword_phrase(query),
|
|
"mentions_count": bool(token_set & {"kolko", "pocet", "spocitaj"}),
|
|
"mentions_shared": (
|
|
mentions_shared
|
|
),
|
|
"mentions_person_answer": (
|
|
mentions_person_answer
|
|
),
|
|
"mentions_thesis": (
|
|
mentions_thesis
|
|
),
|
|
"mentions_work": (
|
|
mentions_work
|
|
),
|
|
"target_kinds": list(
|
|
dict.fromkeys(
|
|
target_kinds
|
|
)
|
|
),
|
|
}
|
|
|
|
|
|
def entity_aliases(
|
|
entity: dict[str, Any],
|
|
) -> list[str]:
|
|
kind = str(
|
|
entity.get(
|
|
"kind"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
values: list[Any] = []
|
|
|
|
if kind in {
|
|
"Person",
|
|
"DocumentAuthor",
|
|
}:
|
|
values.append(
|
|
entity.get(
|
|
"name"
|
|
)
|
|
)
|
|
|
|
elif kind in {
|
|
"Keyword",
|
|
"Topic",
|
|
}:
|
|
values.extend(
|
|
[
|
|
entity.get(
|
|
"name"
|
|
),
|
|
entity.get(
|
|
"id"
|
|
),
|
|
]
|
|
)
|
|
|
|
elif kind == "Category":
|
|
values.append(
|
|
entity.get(
|
|
"name"
|
|
)
|
|
)
|
|
|
|
elif kind in {
|
|
"Thesis",
|
|
"Work",
|
|
}:
|
|
values.append(
|
|
entity.get(
|
|
"title"
|
|
)
|
|
)
|
|
|
|
aliases: list[str] = []
|
|
|
|
for value in values:
|
|
if value is None:
|
|
continue
|
|
|
|
normalized = normalize_for_match(
|
|
str(value)
|
|
)
|
|
|
|
if normalized:
|
|
aliases.append(
|
|
normalized
|
|
)
|
|
|
|
return list(
|
|
dict.fromkeys(
|
|
aliases
|
|
)
|
|
)
|
|
|
|
|
|
def name_token_variants(token: str) -> set[str]:
|
|
"""Conservative Slovak case endings; never accept arbitrary short prefixes."""
|
|
variants = {token}
|
|
if len(token) < 3:
|
|
return variants
|
|
if token.endswith('ova'):
|
|
variants.update(token[:-1] + suffix for suffix in ('ej', 'u', 'ou'))
|
|
elif token.endswith('y'):
|
|
variants.update(token[:-1] + suffix for suffix in ('eho', 'emu', 'ym', 'ych'))
|
|
elif token.endswith('a'):
|
|
variants.update(token[:-1] + suffix for suffix in ('y', 'e', 'u', 'ou', 'ovi', 'om'))
|
|
elif token.endswith('o'):
|
|
variants.update(token[:-1] + suffix for suffix in ('a', 'ovi', 'om'))
|
|
else:
|
|
variants.update(token + suffix for suffix in ('a', 'ovi', 'om'))
|
|
return variants
|
|
|
|
|
|
def person_name_match(query: str, alias: str) -> tuple[float, tuple[int, int] | None]:
|
|
"""Match the complete name in one contiguous window, preserving token order."""
|
|
words, name = normalize_for_match(query).split(), normalize_for_match(alias).split()
|
|
if len(name) < 2:
|
|
return 0.0, None
|
|
best = (0.0, None)
|
|
for start in range(len(words) - len(name) + 1):
|
|
window = words[start:start + len(name)]
|
|
if window == name:
|
|
return 200.0 + len(name), (start, start + len(name))
|
|
if all(observed in name_token_variants(canonical) for canonical, observed in zip(name, window)):
|
|
best = (180.0 + len(name), (start, start + len(name)))
|
|
return best
|
|
|
|
|
|
def requested_name_phrases(query: str) -> list[str]:
|
|
"""Recognise explicitly requested names, including unknown lower-case names.
|
|
|
|
This guard is separate from catalog resolution: an unresolved subject must
|
|
not turn the empty person filter into an unrestricted global listing.
|
|
"""
|
|
normalized = normalize_for_match(query.replace(',', ' separator '))
|
|
candidates = []
|
|
role = re.search(
|
|
r'\b(?:osob(?:a|y|u|e|ach|ami|am|ou)?|student(?:a|ovi|om|i|ov|ka|ky|ke|ku)?|kandidat(?:ov|i|a))\s+(.+)',
|
|
normalized,
|
|
)
|
|
if role:
|
|
tail = re.split(
|
|
r'\s+(?:s(?:o)?\s+(?:nazv\w*|nazov\w*|rok\w*)|splna\w*|ma|maju|mal|mali|je|su|'
|
|
r'ktory\w*|ktora\w*|pri\s+ktorych|v\s+roku|uved\w*|prepojen\w*|spojen\w*|'
|
|
r'vyber|najdi|zisti|cez|pomenuj|uved\w*|vypis|ak)\b', role.group(1), maxsplit=1,
|
|
)[0]
|
|
candidates.extend(re.split(r'\s+(?:a|separator)\s+', tail))
|
|
verb = re.search(r'\b(?:ma|mal|robil|riesil|venuje|venoval)\s+(.+)', normalized)
|
|
if verb:
|
|
candidates.append(re.split(r'\s+(?:v|s|so|a|ak|ktory\w*)\b', verb.group(1), maxsplit=1)[0])
|
|
# Capitalised full names also cover genitive phrases such as 'práce X Y'.
|
|
upper = 'A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ'
|
|
for match in re.finditer(rf'\b([{upper}][^\W\d_]*(?:\s+[{upper}][^\W\d_]*){{1,3}})\b', query):
|
|
words = match.group(1).split()
|
|
if all(not word.isupper() for word in words):
|
|
candidates.append(normalize_for_match(match.group(1)))
|
|
generic = {'ktore', 'ktori', 'ktora', 'ktory', 'vsetci', 'vsetky', 'vsetkych', 'maju', 'ma',
|
|
'praca', 'prace', 'pracu', 'projekt', 'projekty', 'dokument', 'dokumenty', 'rok', 'roku',
|
|
'klucove', 'klucovy', 'klucovym', 'tag', 'tagy', 'nazov', 'nazvy', 'temu',
|
|
'kategoriu', 'kategorie', 'uvedenu', 'uvedeny', 'priradeny', 'priradene',
|
|
'je', 'su', 'sa', 's', 'pri', 'pre', 'z', 'na', 'v', 'ake', 'aky', 'kto', 'kolko'}
|
|
return list(dict.fromkeys(
|
|
candidate.strip() for candidate in candidates
|
|
if 1 <= len(candidate.split()) <= 5 and candidate.split()[0] not in generic
|
|
))
|
|
|
|
|
|
def named_author_phrases(query: str) -> list[str]:
|
|
normalized = normalize_for_match(query.replace(',', ' separator '))
|
|
match = re.search(r'\bautor(?:a|om)?(?:\s+dokument(?:u|ov))?\s+(.+)', normalized)
|
|
if not match:
|
|
return []
|
|
name = re.split(r'\s+(?:v|s|so|a|ak|ktory\w*|pri|spojen\w*)\b', match.group(1), maxsplit=1)[0]
|
|
return [name] if 2 <= len(name.split()) <= 4 and name.split()[0] not in {
|
|
'dokumentu', 'dokumentov', 'je', 'sa', 'pri', 'osoby', 'osobe', 'osobu', 'osob',
|
|
'ktory', 'ktora', 'ktoreho', 'studentov', 'studenta',
|
|
} else []
|
|
|
|
|
|
def explicit_keyword_entities(phrase: str, catalog: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Resolve a requested tag as a complete leading phrase, not unrelated words."""
|
|
normalized = normalize_for_match(phrase)
|
|
matches = []
|
|
for entity in catalog:
|
|
if entity.get('kind') != 'Keyword':
|
|
continue
|
|
aliases = entity_aliases(entity)
|
|
matching = [alias for alias in aliases if normalized == alias]
|
|
if matching:
|
|
best = max(matching, key=len)
|
|
matches.append({**entity, 'matched_alias': best, 'match_score': 200.0 + len(best)})
|
|
if not matches:
|
|
return []
|
|
# Prefer the longest full tag when one tag is a prefix of another.
|
|
longest = max(len(row['matched_alias']) for row in matches)
|
|
return [row for row in matches if len(row['matched_alias']) == longest]
|
|
|
|
|
|
def tokens_match(
|
|
left: str,
|
|
right: str,
|
|
) -> bool:
|
|
if left == right:
|
|
return True
|
|
|
|
shorter = min(
|
|
len(left),
|
|
len(right),
|
|
)
|
|
|
|
if shorter <= 2:
|
|
return False
|
|
|
|
required = (
|
|
3
|
|
if shorter <= 4
|
|
else 4
|
|
if shorter <= 6
|
|
else 5
|
|
)
|
|
|
|
return (
|
|
left[:required]
|
|
== right[:required]
|
|
)
|
|
|
|
|
|
def alias_match_score(
|
|
normalized_query: str,
|
|
query_tokens: set[str],
|
|
alias: str,
|
|
) -> float:
|
|
alias_tokens = (
|
|
alias.split()
|
|
)
|
|
|
|
if not alias_tokens:
|
|
return 0.0
|
|
|
|
if len(
|
|
alias_tokens
|
|
) == 1:
|
|
if alias not in query_tokens:
|
|
return 0.0
|
|
|
|
return (
|
|
50.0
|
|
+ min(
|
|
len(alias),
|
|
20,
|
|
)
|
|
/ 100.0
|
|
)
|
|
|
|
padded_query = (
|
|
f" {normalized_query} "
|
|
)
|
|
|
|
padded_alias = (
|
|
f" {alias} "
|
|
)
|
|
|
|
if (
|
|
padded_alias
|
|
in padded_query
|
|
):
|
|
score = (
|
|
100.0
|
|
+ len(
|
|
alias_tokens
|
|
)
|
|
* 5.0
|
|
+ min(
|
|
len(alias),
|
|
100,
|
|
)
|
|
/ 100.0
|
|
)
|
|
|
|
if (
|
|
normalized_query
|
|
== alias
|
|
):
|
|
score += 50.0
|
|
|
|
return score
|
|
|
|
if all(
|
|
any(
|
|
tokens_match(
|
|
alias_token,
|
|
query_token,
|
|
)
|
|
for query_token
|
|
in query_tokens
|
|
)
|
|
for alias_token
|
|
in alias_tokens
|
|
):
|
|
return (
|
|
95.0
|
|
+ len(
|
|
alias_tokens
|
|
)
|
|
* 5.0
|
|
+ min(
|
|
len(alias),
|
|
100,
|
|
)
|
|
/ 100.0
|
|
)
|
|
|
|
return 0.0
|
|
|
|
|
|
def suppress_nested_entity_matches(
|
|
matches: list[
|
|
dict[str, Any]
|
|
],
|
|
) -> list[dict[str, Any]]:
|
|
result: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
for item in matches:
|
|
kind = str(
|
|
item.get(
|
|
"kind"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
if kind not in {
|
|
"Thesis",
|
|
"Work",
|
|
}:
|
|
result.append(
|
|
item
|
|
)
|
|
continue
|
|
|
|
alias = str(
|
|
item.get(
|
|
"matched_alias"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
score = float(
|
|
item.get(
|
|
"match_score"
|
|
)
|
|
or 0.0
|
|
)
|
|
|
|
dominated = False
|
|
|
|
for other in matches:
|
|
if other is item:
|
|
continue
|
|
|
|
if (
|
|
other.get(
|
|
"kind"
|
|
)
|
|
!= kind
|
|
):
|
|
continue
|
|
|
|
other_alias = str(
|
|
other.get(
|
|
"matched_alias"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
other_score = float(
|
|
other.get(
|
|
"match_score"
|
|
)
|
|
or 0.0
|
|
)
|
|
|
|
if (
|
|
not alias
|
|
or not other_alias
|
|
or alias == other_alias
|
|
):
|
|
continue
|
|
|
|
if (
|
|
alias in other_alias
|
|
and len(
|
|
other_alias
|
|
)
|
|
> len(
|
|
alias
|
|
)
|
|
and other_score
|
|
> score
|
|
):
|
|
dominated = True
|
|
break
|
|
|
|
if not dominated:
|
|
result.append(
|
|
item
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def resolve_entities(
|
|
query: str,
|
|
catalog: list[
|
|
dict[str, Any]
|
|
],
|
|
*,
|
|
max_entities: int = (
|
|
DEFAULT_MAX_ENTITIES
|
|
),
|
|
preferred_kinds: Iterable[
|
|
str
|
|
] = (),
|
|
) -> list[dict[str, Any]]:
|
|
normalized_query = (
|
|
normalize_for_match(
|
|
query
|
|
)
|
|
)
|
|
|
|
query_tokens = set(
|
|
normalized_query.split()
|
|
)
|
|
|
|
preferred = set(
|
|
preferred_kinds
|
|
)
|
|
|
|
matches: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
for entity in catalog:
|
|
best_score = 0.0
|
|
best_alias = ""
|
|
|
|
for alias in entity_aliases(
|
|
entity
|
|
):
|
|
name_span = None
|
|
if entity.get("kind") in {"Person", "DocumentAuthor"}:
|
|
score, name_span = person_name_match(normalized_query, alias)
|
|
else:
|
|
score = alias_match_score(
|
|
normalized_query,
|
|
query_tokens,
|
|
alias,
|
|
)
|
|
|
|
if (
|
|
score > 0
|
|
and entity.get(
|
|
"kind"
|
|
)
|
|
in preferred
|
|
):
|
|
score += 2.0
|
|
|
|
if score > best_score:
|
|
best_score = score
|
|
best_alias = alias
|
|
best_name_span = name_span
|
|
|
|
if best_score <= 0.0:
|
|
continue
|
|
|
|
item = dict(
|
|
entity
|
|
)
|
|
|
|
item[
|
|
"matched_alias"
|
|
] = best_alias
|
|
if entity.get("kind") in {"Person", "DocumentAuthor"}:
|
|
item["matched_name_span"] = best_name_span
|
|
|
|
item[
|
|
"match_score"
|
|
] = round(
|
|
best_score,
|
|
3,
|
|
)
|
|
|
|
matches.append(
|
|
item
|
|
)
|
|
|
|
exact_name_spans = {
|
|
tuple(item["matched_name_span"])
|
|
for item in matches
|
|
if item.get("matched_name_span") is not None and item["match_score"] >= 200
|
|
}
|
|
matches = [item for item in matches if not (
|
|
item.get("matched_name_span") is not None
|
|
and tuple(item["matched_name_span"]) in exact_name_spans
|
|
and item["match_score"] < 200
|
|
)]
|
|
|
|
person_aliases = {
|
|
str(
|
|
item.get(
|
|
"matched_alias"
|
|
)
|
|
or ""
|
|
)
|
|
for item in matches
|
|
if item.get(
|
|
"kind"
|
|
)
|
|
== "Person"
|
|
}
|
|
|
|
matches = [
|
|
item
|
|
for item in matches
|
|
if not (
|
|
item.get(
|
|
"kind"
|
|
)
|
|
== "DocumentAuthor"
|
|
and str(
|
|
item.get(
|
|
"matched_alias"
|
|
)
|
|
or ""
|
|
)
|
|
in person_aliases
|
|
)
|
|
]
|
|
|
|
matches = (
|
|
suppress_nested_entity_matches(
|
|
matches
|
|
)
|
|
)
|
|
|
|
matches.sort(
|
|
key=lambda item: (
|
|
-float(
|
|
item[
|
|
"match_score"
|
|
]
|
|
),
|
|
str(
|
|
item.get(
|
|
"kind"
|
|
)
|
|
or ""
|
|
),
|
|
str(
|
|
item.get(
|
|
"name"
|
|
)
|
|
or item.get(
|
|
"title"
|
|
)
|
|
or item.get(
|
|
"id"
|
|
)
|
|
or ""
|
|
),
|
|
)
|
|
)
|
|
|
|
selected: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
seen_ids: set[
|
|
str
|
|
] = set()
|
|
|
|
for item in matches:
|
|
element_id = str(
|
|
item.get(
|
|
"element_id"
|
|
)
|
|
or ""
|
|
)
|
|
|
|
if not element_id:
|
|
continue
|
|
|
|
if element_id in seen_ids:
|
|
continue
|
|
|
|
seen_ids.add(
|
|
element_id
|
|
)
|
|
|
|
selected.append(
|
|
item
|
|
)
|
|
|
|
if (
|
|
len(selected)
|
|
>= max_entities
|
|
):
|
|
break
|
|
|
|
return selected
|
|
|
|
|
|
def rows_from_result(
|
|
result: Any,
|
|
) -> list[dict[str, Any]]:
|
|
rows: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
for record in result:
|
|
if isinstance(
|
|
record,
|
|
dict,
|
|
):
|
|
rows.append(
|
|
dict(
|
|
record
|
|
)
|
|
)
|
|
continue
|
|
|
|
data = getattr(
|
|
record,
|
|
"data",
|
|
None,
|
|
)
|
|
|
|
if callable(
|
|
data
|
|
):
|
|
rows.append(
|
|
dict(
|
|
data()
|
|
)
|
|
)
|
|
continue
|
|
|
|
rows.append(
|
|
dict(
|
|
record
|
|
)
|
|
)
|
|
|
|
return rows
|
|
|
|
|
|
def run_rows(
|
|
session: Any,
|
|
query: str,
|
|
**parameters: Any,
|
|
) -> list[dict[str, Any]]:
|
|
return rows_from_result(
|
|
session.run(
|
|
query,
|
|
**parameters,
|
|
)
|
|
)
|
|
|
|
|
|
def deduplicate_paths(
|
|
paths: list[
|
|
dict[str, Any]
|
|
],
|
|
) -> list[dict[str, Any]]:
|
|
result: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
seen: set[
|
|
str
|
|
] = set()
|
|
|
|
for path in paths:
|
|
key = json.dumps(
|
|
{
|
|
"nodes": path.get(
|
|
"nodes",
|
|
[],
|
|
),
|
|
"relationships": (
|
|
path.get(
|
|
"relationships",
|
|
[],
|
|
)
|
|
),
|
|
},
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
default=str,
|
|
)
|
|
|
|
if key in seen:
|
|
continue
|
|
|
|
seen.add(
|
|
key
|
|
)
|
|
|
|
result.append(
|
|
path
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def deduplicate_rows(
|
|
rows: list[
|
|
dict[str, Any]
|
|
],
|
|
key_fields: tuple[
|
|
str,
|
|
...,
|
|
],
|
|
) -> list[dict[str, Any]]:
|
|
result: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
seen: set[
|
|
tuple[str, ...]
|
|
] = set()
|
|
|
|
for row in rows:
|
|
key = tuple(
|
|
str(
|
|
row.get(
|
|
field
|
|
)
|
|
or ""
|
|
)
|
|
for field
|
|
in key_fields
|
|
)
|
|
|
|
if key in seen:
|
|
continue
|
|
|
|
seen.add(
|
|
key
|
|
)
|
|
|
|
result.append(
|
|
row
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def collect_documents(
|
|
resolved_entities: list[
|
|
dict[str, Any]
|
|
],
|
|
paths: list[
|
|
dict[str, Any]
|
|
],
|
|
facts: list[
|
|
dict[str, Any]
|
|
],
|
|
) -> list[dict[str, Any]]:
|
|
documents: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
def add_document(
|
|
path: Any,
|
|
source_url: Any = None,
|
|
) -> None:
|
|
value = str(
|
|
path
|
|
or ""
|
|
).strip()
|
|
|
|
if not value:
|
|
return
|
|
|
|
current = (
|
|
documents.setdefault(
|
|
value,
|
|
{
|
|
"document_path": (
|
|
value
|
|
),
|
|
"source_url": None,
|
|
},
|
|
)
|
|
)
|
|
|
|
if (
|
|
source_url
|
|
and not current.get(
|
|
"source_url"
|
|
)
|
|
):
|
|
current[
|
|
"source_url"
|
|
] = str(
|
|
source_url
|
|
)
|
|
|
|
for entity in (
|
|
resolved_entities
|
|
):
|
|
add_document(
|
|
entity.get(
|
|
"source_document"
|
|
)
|
|
)
|
|
|
|
for path in paths:
|
|
for node in path.get(
|
|
"nodes",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
node,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
if (
|
|
node.get(
|
|
"kind"
|
|
)
|
|
== "Document"
|
|
):
|
|
add_document(
|
|
node.get(
|
|
"path"
|
|
),
|
|
node.get(
|
|
"source_url"
|
|
),
|
|
)
|
|
|
|
add_document(
|
|
node.get(
|
|
"source_document"
|
|
)
|
|
)
|
|
|
|
for fact in facts:
|
|
add_document(
|
|
fact.get(
|
|
"document_path"
|
|
),
|
|
fact.get(
|
|
"source_url"
|
|
),
|
|
)
|
|
|
|
add_document(
|
|
fact.get(
|
|
"left_document_path"
|
|
),
|
|
fact.get(
|
|
"left_source_url"
|
|
),
|
|
)
|
|
|
|
add_document(
|
|
fact.get(
|
|
"right_document_path"
|
|
),
|
|
fact.get(
|
|
"right_source_url"
|
|
),
|
|
)
|
|
|
|
return sorted(
|
|
documents.values(),
|
|
key=lambda item: (
|
|
item[
|
|
"document_path"
|
|
]
|
|
),
|
|
)
|
|
|
|
|
|
def build_facts(
|
|
thesis_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
work_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
start_year_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
person_keyword_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
person_category_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
shared_keyword_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
shared_category_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
shared_document_author_rows: list[
|
|
dict[str, Any]
|
|
],
|
|
*,
|
|
mentions_shared: bool,
|
|
) -> list[dict[str, Any]]:
|
|
facts: list[
|
|
dict[str, Any]
|
|
] = []
|
|
|
|
for row in thesis_rows:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "person_thesis"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
for row in work_rows:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "person_work"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
unique_start_rows = (
|
|
deduplicate_rows(
|
|
start_year_rows,
|
|
(
|
|
"person_element_id",
|
|
),
|
|
)
|
|
)
|
|
|
|
for row in unique_start_rows:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "person_start_year"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
unique_person_keywords = (
|
|
deduplicate_rows(
|
|
person_keyword_rows,
|
|
(
|
|
"person_element_id",
|
|
"keyword_element_id",
|
|
),
|
|
)
|
|
)
|
|
|
|
for row in unique_person_keywords:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "person_keyword"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
unique_person_categories = (
|
|
deduplicate_rows(
|
|
person_category_rows,
|
|
(
|
|
"person_element_id",
|
|
"category_element_id",
|
|
),
|
|
)
|
|
)
|
|
|
|
for row in unique_person_categories:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "person_category"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
unique_shared_keywords = (
|
|
deduplicate_rows(
|
|
shared_keyword_rows,
|
|
(
|
|
"left_person_element_id",
|
|
"right_person_element_id",
|
|
"keyword_element_id",
|
|
),
|
|
)
|
|
)
|
|
|
|
for row in unique_shared_keywords:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "shared_keyword"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
unique_shared_categories = (
|
|
deduplicate_rows(
|
|
shared_category_rows,
|
|
(
|
|
"left_person_element_id",
|
|
"right_person_element_id",
|
|
"category_element_id",
|
|
),
|
|
)
|
|
)
|
|
|
|
for row in unique_shared_categories:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = "shared_category"
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
unique_shared_authors = (
|
|
deduplicate_rows(
|
|
shared_document_author_rows,
|
|
(
|
|
"left_person_element_id",
|
|
"right_person_element_id",
|
|
"document_author_element_id",
|
|
),
|
|
)
|
|
)
|
|
|
|
for row in unique_shared_authors:
|
|
fact = dict(
|
|
row
|
|
)
|
|
|
|
fact[
|
|
"fact_type"
|
|
] = (
|
|
"shared_document_author"
|
|
)
|
|
|
|
facts.append(
|
|
fact
|
|
)
|
|
|
|
if (
|
|
mentions_shared
|
|
and len(
|
|
unique_start_rows
|
|
)
|
|
>= 2
|
|
):
|
|
ordered = sorted(
|
|
unique_start_rows,
|
|
key=lambda row: str(
|
|
row.get(
|
|
"person"
|
|
)
|
|
or ""
|
|
),
|
|
)
|
|
|
|
years = [
|
|
row.get(
|
|
"start_year"
|
|
)
|
|
for row in ordered
|
|
]
|
|
|
|
known_years = [
|
|
year
|
|
for year in years
|
|
if year is not None
|
|
]
|
|
|
|
same = (
|
|
len(
|
|
known_years
|
|
)
|
|
== len(
|
|
years
|
|
)
|
|
and len(
|
|
set(
|
|
known_years
|
|
)
|
|
)
|
|
== 1
|
|
)
|
|
|
|
facts.append(
|
|
{
|
|
"fact_type": (
|
|
"start_year_comparison"
|
|
),
|
|
"people": [
|
|
str(
|
|
row.get(
|
|
"person"
|
|
)
|
|
or ""
|
|
)
|
|
for row in ordered
|
|
],
|
|
"start_years": {
|
|
str(
|
|
row.get(
|
|
"person"
|
|
)
|
|
or ""
|
|
): row.get(
|
|
"start_year"
|
|
)
|
|
for row in ordered
|
|
},
|
|
"same_start_year": (
|
|
same
|
|
),
|
|
"shared_start_year": (
|
|
known_years[0]
|
|
if same
|
|
else None
|
|
),
|
|
}
|
|
)
|
|
|
|
return facts
|
|
|
|
|
|
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY = """
|
|
MATCH (person:Person)-[:HAS_DOCUMENT]->(document:Document)
|
|
MATCH (author:DocumentAuthor)-[:AUTHORED_DOCUMENT]->(document)
|
|
WHERE person.graph_scope = $scope AND document.graph_scope = $scope
|
|
AND author.graph_scope = $scope
|
|
AND (size($person_ids) = 0 OR elementId(person) IN $person_ids)
|
|
AND (size($author_ids) = 0 OR elementId(author) IN $author_ids)
|
|
OPTIONAL MATCH (document)-[:IN_CATEGORY]->(category:Category)
|
|
WHERE category.graph_scope = $scope
|
|
WITH person, document, author, category
|
|
WHERE size($category_ids) = 0 OR elementId(category) IN $category_ids
|
|
RETURN DISTINCT elementId(person) AS person_element_id, person.name AS person,
|
|
elementId(author) AS document_author_element_id, author.name AS document_author,
|
|
elementId(category) AS category_element_id, category.name AS category,
|
|
document.path AS document_path, document.source_url AS source_url
|
|
ORDER BY person, document_author, document_path
|
|
LIMIT $limit
|
|
"""
|
|
|
|
DOCUMENT_SOURCES_QUERY = """
|
|
MATCH (document:Document)
|
|
WHERE document.graph_scope = $scope AND document.path IN $paths
|
|
RETURN document.path AS document_path, document.source_url AS source_url
|
|
ORDER BY document_path
|
|
"""
|
|
|
|
|
|
def search_graph_with_session(
|
|
session: Any,
|
|
query: str,
|
|
*,
|
|
limit: int = DEFAULT_LIMIT,
|
|
max_entities: int = DEFAULT_MAX_ENTITIES,
|
|
) -> dict[str, Any]:
|
|
clean_query = str(query or "").strip()
|
|
if not clean_query:
|
|
raise ValueError("Query nesmie byť prázdny")
|
|
if limit < 1:
|
|
raise ValueError("Limit musí byť aspoň 1")
|
|
if max_entities < 1:
|
|
raise ValueError("max_entities musí byť aspoň 1")
|
|
|
|
constraints = extract_query_constraints(clean_query)
|
|
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE)
|
|
# Explicit tag filters are resolved separately. Incidental names such as the
|
|
# tag 'zp' in 'ZP Wiki' must never become additional query constraints.
|
|
phrase = constraints["keyword_phrase"]
|
|
entities = resolve_entities(
|
|
clean_query, catalog, max_entities=max(len(catalog), max_entities),
|
|
preferred_kinds=constraints["target_kinds"],
|
|
)
|
|
if phrase:
|
|
keywords = explicit_keyword_entities(phrase, catalog)
|
|
entities = [row for row in entities if row.get("kind") != "Keyword"] + keywords
|
|
constraints["unresolved_keyword"] = not bool(keywords)
|
|
else:
|
|
constraints["unresolved_keyword"] = False
|
|
if constraints["mentions_keyword"]:
|
|
entities = [row for row in entities if row.get("kind") != "Keyword"]
|
|
elif "zp wiki" in normalize_for_match(clean_query):
|
|
entities = [row for row in entities if not (
|
|
row.get("kind") == "Keyword" and row.get("matched_alias") == "zp"
|
|
)]
|
|
|
|
author_mentions = named_author_phrases(clean_query) if constraints["mentions_document_author"] else []
|
|
requested_authors = [
|
|
{**row, "matched_alias": normalize_for_match(row.get("name", "")), "match_score": score}
|
|
for row in catalog if row.get("kind") == "DocumentAuthor"
|
|
for requested in author_mentions
|
|
for score, _ in [person_name_match(requested, row.get("name", ""))]
|
|
if score > 0
|
|
]
|
|
if author_mentions:
|
|
author_names = {row["matched_alias"] for row in requested_authors}
|
|
entities = [row for row in entities if row.get("kind") != "DocumentAuthor" and not (
|
|
row.get("kind") == "Person" and row.get("matched_alias") in author_names
|
|
)] + requested_authors
|
|
|
|
requested_names = requested_name_phrases(clean_query)
|
|
unknown_names = []
|
|
for requested in requested_names:
|
|
if phrase and normalize_for_match(requested) in normalize_for_match(phrase):
|
|
continue
|
|
personal = [row for row in catalog if row.get("kind") in {"Person", "DocumentAuthor"}]
|
|
known_person = any(person_name_match(requested, row.get("name", ""))[0] > 0 for row in personal)
|
|
known_title = any(
|
|
normalize_for_match(requested) in alias
|
|
for row in catalog if row.get("kind") not in {"Person", "DocumentAuthor"}
|
|
for alias in entity_aliases(row)
|
|
)
|
|
if len(requested.split()) == 1:
|
|
# A verb can precede the first inflected word of a fully resolved
|
|
# thesis title. That is a title anchor, not an unknown person.
|
|
known_title = known_title or any(
|
|
tokens_match(normalize_for_match(requested), word)
|
|
for row in entities if row.get('kind') in {'Thesis', 'Work'}
|
|
for alias in entity_aliases(row) for word in alias.split()
|
|
)
|
|
if not known_person and not known_title:
|
|
unknown_names.append(requested)
|
|
constraints["requested_person_names"] = requested_names
|
|
constraints["unresolved_person_names"] = unknown_names
|
|
constraints["unresolved_document_author"] = bool(author_mentions and not requested_authors)
|
|
aliases_to_ids: dict[tuple[str, str], set[str]] = {}
|
|
for row in entities:
|
|
if row.get("kind") in {"Person", "DocumentAuthor"}:
|
|
key = (row["kind"], row["matched_alias"])
|
|
aliases_to_ids.setdefault(key, set()).add(str(row["element_id"]))
|
|
ambiguous_names = [name for (_, name), matches in aliases_to_ids.items() if len(matches) > 1]
|
|
constraints["ambiguous_person_names"] = sorted(set(ambiguous_names))
|
|
|
|
# Preserve explicitly mentioned people when a longer topic/title also matches.
|
|
entities.sort(key=lambda row: (row.get("kind") != "Person", -row["match_score"]))
|
|
resolved_entities = entities[:max_entities]
|
|
entity_limit_reached = len(entities) > max_entities
|
|
|
|
def ids(kind: str) -> list[str]:
|
|
return [str(row["element_id"]) for row in resolved_entities
|
|
if row.get("kind") == kind and row.get("element_id")]
|
|
|
|
person_ids = ids("Person")
|
|
thesis_ids, work_ids = ids("Thesis"), ids("Work")
|
|
keyword_ids, category_ids = ids("Keyword"), ids("Category")
|
|
author_ids = ids("DocumentAuthor")
|
|
year_specific = bool(constraints["years"] or constraints["mentions_academic_year"])
|
|
title_specific = constraints["mentions_title"]
|
|
thesis_specific = bool(
|
|
constraints["mentions_thesis"] or thesis_ids
|
|
or ((title_specific or year_specific) and not constraints["mentions_work"])
|
|
) and not constraints["mentions_start_year"]
|
|
work_specific = bool(
|
|
constraints["mentions_work"] or work_ids
|
|
or ((title_specific or year_specific) and not constraints["mentions_thesis"])
|
|
) and not constraints["mentions_start_year"]
|
|
shared_keyword = bool(
|
|
constraints["mentions_shared"] and len(person_ids) >= 2
|
|
and (constraints["mentions_keyword"] or constraints["mentions_topic"])
|
|
)
|
|
shared_category = bool(
|
|
constraints["mentions_shared"] and len(person_ids) >= 2
|
|
and constraints["mentions_category"]
|
|
)
|
|
shared_author = bool(
|
|
constraints["mentions_shared"] and len(person_ids) >= 2
|
|
and constraints["mentions_document_author"]
|
|
)
|
|
keyword_specific = bool(
|
|
not shared_keyword and (
|
|
(keyword_ids and not (thesis_specific or work_specific))
|
|
or (person_ids and constraints["mentions_keyword"])
|
|
)
|
|
)
|
|
category_specific = bool(
|
|
not shared_category and (category_ids or (person_ids and constraints["mentions_category"]))
|
|
)
|
|
author_specific = bool(not shared_author and constraints["mentions_document_author"])
|
|
structured = any((
|
|
thesis_specific, work_specific, constraints["mentions_start_year"],
|
|
keyword_specific, category_specific, author_specific,
|
|
shared_keyword, shared_category, shared_author,
|
|
))
|
|
# Fetch complete structured evidence before filtering/counting and applying
|
|
# the presentation limit. The catalog bounds the number of academic nodes;
|
|
# relational lookups use a conservative bound covering the managed graph.
|
|
fetch_limit = max(limit, len(catalog) ** 2)
|
|
common = {"scope": GRAPH_SCOPE, "person_ids": person_ids, "limit": fetch_limit}
|
|
blocked_reasons = []
|
|
if constraints["unresolved_keyword"]:
|
|
blocked_reasons.append("unknown_keyword")
|
|
if unknown_names:
|
|
blocked_reasons.append("unknown_person")
|
|
if constraints["unresolved_document_author"]:
|
|
blocked_reasons.append("unknown_document_author")
|
|
if ambiguous_names:
|
|
blocked_reasons.append("ambiguous_person")
|
|
if constraints["mentions_count"] and not (thesis_specific or work_specific):
|
|
blocked_reasons.append("unsupported_count_domain")
|
|
# A quoted unknown work title is a failed exact lookup, not a global listing.
|
|
quoted_titles = re.findall(r'[„“"«]([^„“”"»]+)[“”"»]', clean_query)
|
|
if quoted_titles and (thesis_specific or work_specific) and not thesis_ids and not work_ids:
|
|
blocked_reasons.append("unknown_title")
|
|
constraints["unresolved_title"] = True
|
|
blocked = bool(blocked_reasons)
|
|
|
|
def fetch(enabled: bool, cypher: str, **params: Any) -> list[dict[str, Any]]:
|
|
return run_rows(session, cypher, **common, **params) if enabled and not blocked else []
|
|
|
|
thesis_rows = fetch(thesis_specific, THESIS_FACTS_QUERY,
|
|
thesis_ids=thesis_ids, years=constraints["years"],
|
|
thesis_types=constraints["thesis_types"])
|
|
work_rows = fetch(work_specific, WORK_FACTS_QUERY,
|
|
work_ids=work_ids, years=constraints["years"],
|
|
work_types=constraints["work_types"])
|
|
start_rows = fetch(constraints["mentions_start_year"], START_YEAR_FACTS_QUERY,
|
|
years=constraints["years"])
|
|
keyword_rows = fetch(keyword_specific, PERSON_KEYWORD_FACTS_QUERY, keyword_ids=keyword_ids)
|
|
category_rows = fetch(category_specific, PERSON_CATEGORY_FACTS_QUERY, category_ids=category_ids)
|
|
author_rows = fetch(author_specific, PERSON_DOCUMENT_AUTHOR_FACTS_QUERY,
|
|
author_ids=author_ids, category_ids=category_ids)
|
|
shared_keyword_rows = fetch(shared_keyword, SHARED_KEYWORD_FACTS_QUERY)
|
|
if keyword_ids:
|
|
shared_keyword_rows = [row for row in shared_keyword_rows
|
|
if str(row.get("keyword_element_id")) in keyword_ids]
|
|
shared_category_rows = fetch(shared_category, SHARED_CATEGORY_FACTS_QUERY)
|
|
if category_ids:
|
|
shared_category_rows = [row for row in shared_category_rows
|
|
if str(row.get("category_element_id")) in category_ids]
|
|
shared_author_rows = fetch(shared_author, SHARED_DOCUMENT_AUTHOR_FACTS_QUERY)
|
|
|
|
# Conjunctions must be witnessed by the same person, not by unrelated rows.
|
|
if keyword_specific and (thesis_specific or work_specific):
|
|
matching_people = {row.get("person_element_id") for row in keyword_rows}
|
|
thesis_rows = [row for row in thesis_rows if row.get("person_element_id") in matching_people]
|
|
work_rows = [row for row in work_rows if row.get("person_element_id") in matching_people]
|
|
academic_people = {row.get("person_element_id") for row in thesis_rows + work_rows}
|
|
keyword_rows = [row for row in keyword_rows if row.get("person_element_id") in academic_people]
|
|
|
|
shared_years: list[int] | None = None
|
|
if (not blocked and constraints["mentions_academic_year"] and constraints["mentions_shared"]
|
|
and len(person_ids) >= 2 and not (shared_keyword or shared_category or shared_author)):
|
|
years_by_person = {
|
|
person: {row["year"] for row in thesis_rows + work_rows
|
|
if row.get("person_element_id") == person and row.get("year") is not None}
|
|
for person in person_ids
|
|
}
|
|
shared_years = sorted(set.intersection(*years_by_person.values()))
|
|
thesis_rows = [row for row in thesis_rows if row.get("year") in shared_years]
|
|
work_rows = [row for row in work_rows if row.get("year") in shared_years]
|
|
|
|
paths: list[dict[str, Any]] = []
|
|
for enabled, cypher, rows, kind, node_ids, types in (
|
|
(thesis_specific, THESIS_PATHS_QUERY, thesis_rows, "Thesis", thesis_ids, constraints["thesis_types"]),
|
|
(work_specific, WORK_PATHS_QUERY, work_rows, "Work", work_ids, constraints["work_types"]),
|
|
):
|
|
params = ({"thesis_ids": node_ids, "thesis_types": types} if kind == "Thesis"
|
|
else {"work_ids": node_ids, "work_types": types})
|
|
returned = fetch(enabled, cypher, years=constraints["years"], **params)
|
|
allowed = {str(row.get("thesis_element_id") or row.get("work_element_id")) for row in rows}
|
|
paths.extend(path for path in returned if any(
|
|
node.get("kind") == kind and str(node.get("element_id")) in allowed
|
|
for node in path.get("nodes", [])
|
|
))
|
|
if not structured and resolved_entities and not blocked:
|
|
start_ids = [str(row["element_id"]) for row in resolved_entities]
|
|
params = {"scope": GRAPH_SCOPE, "start_ids": start_ids, "limit": limit}
|
|
if constraints["mentions_shared"] and len(start_ids) >= 2:
|
|
paths.extend(run_rows(session, PAIR_PATH_QUERY, **params))
|
|
else:
|
|
paths.extend(run_rows(session, NEIGHBORHOOD_QUERY,
|
|
target_kinds=constraints["target_kinds"], **params))
|
|
paths = deduplicate_paths(paths)
|
|
facts = build_facts(
|
|
thesis_rows, work_rows, start_rows, keyword_rows, category_rows,
|
|
shared_keyword_rows, shared_category_rows, shared_author_rows,
|
|
mentions_shared=constraints["mentions_shared"],
|
|
)
|
|
for row in author_rows:
|
|
facts.append({**row, "fact_type": "person_document_author"})
|
|
if row.get("category") and not any(
|
|
fact.get("fact_type") == "person_category"
|
|
and fact.get("person_element_id") == row.get("person_element_id")
|
|
and fact.get("document_path") == row.get("document_path")
|
|
and fact.get("category") == row.get("category")
|
|
for fact in facts
|
|
):
|
|
facts.append({**row, "fact_type": "person_category"})
|
|
if shared_years is not None:
|
|
facts.append({"fact_type": "academic_year_comparison", "shared_years": shared_years,
|
|
"same_academic_year": bool(shared_years),
|
|
"people": [row["name"] for row in resolved_entities if row.get("kind") == "Person"]})
|
|
|
|
aggregation = None
|
|
if constraints["mentions_count"] and not blocked:
|
|
matching = sorted({str(row["person"]) for row in thesis_rows + work_rows})
|
|
aggregation = {"count": len(matching), "people": matching,
|
|
"candidate_count": len(person_ids), "complete": not entity_limit_reached}
|
|
# Sources of rejected candidates still document the comparison. Fetch their
|
|
# actual stored URLs rather than manufacturing URLs from expected answers.
|
|
documents = collect_documents(resolved_entities, paths, facts)
|
|
sources = run_rows(session, DOCUMENT_SOURCES_QUERY, scope=GRAPH_SCOPE,
|
|
paths=[row["document_path"] for row in documents]) if documents else []
|
|
by_path = {row["document_path"]: row.get("source_url") for row in sources}
|
|
for document in documents:
|
|
if not document.get("source_url"):
|
|
document["source_url"] = by_path.get(document["document_path"])
|
|
|
|
strategies = []
|
|
for active, name in (
|
|
(resolved_entities, "entity_resolution"), (paths, "graph_traversal"),
|
|
(thesis_rows, "thesis_constraints"), (work_rows, "work_constraints"),
|
|
(start_rows, "start_year_constraints"), (keyword_rows, "person_keyword_constraints"),
|
|
(category_rows, "person_category_constraints"), (author_rows, "person_document_author_constraints"),
|
|
(shared_keyword_rows, "shared_keyword_constraints"),
|
|
(shared_category_rows, "shared_category_constraints"),
|
|
(shared_author_rows, "shared_document_author_constraints"),
|
|
):
|
|
if active:
|
|
strategies.append(name)
|
|
positive_facts = [row for row in facts if row.get("fact_type") not in {
|
|
"academic_year_comparison", "start_year_comparison"
|
|
}]
|
|
status = "ambiguous" if ambiguous_names else (
|
|
"no_answer" if blocked or not (positive_facts or paths or aggregation is not None) else "ok"
|
|
)
|
|
reason = blocked_reasons[0] if blocked_reasons else (
|
|
"no_matching_facts" if status == "no_answer" else None
|
|
)
|
|
return {
|
|
"status": status, "reason": reason, "reasons": blocked_reasons,
|
|
"query": clean_query, "strategies": strategies, "constraints": constraints,
|
|
"resolved_entities": resolved_entities, "paths": paths[:limit], "facts": facts[:limit],
|
|
"documents": documents, "aggregation": aggregation,
|
|
"checked_entities": resolved_entities,
|
|
"rejection_evidence": [row for row in facts if row.get("fact_type") in {
|
|
"academic_year_comparison", "start_year_comparison"
|
|
} and not positive_facts],
|
|
"truncated": entity_limit_reached or len(facts) > limit or len(paths) > limit,
|
|
}
|
|
|
|
|
|
|
|
|
|
def search_graph(
|
|
query: str,
|
|
*,
|
|
limit: int = (
|
|
DEFAULT_LIMIT
|
|
),
|
|
max_entities: int = (
|
|
DEFAULT_MAX_ENTITIES
|
|
),
|
|
) -> dict[str, Any]:
|
|
from .graph_db import (
|
|
create_neo4j_driver,
|
|
get_neo4j_settings,
|
|
)
|
|
|
|
settings = (
|
|
get_neo4j_settings()
|
|
)
|
|
|
|
with create_neo4j_driver(
|
|
settings
|
|
) as driver:
|
|
driver.verify_connectivity()
|
|
|
|
with driver.session(
|
|
database=(
|
|
settings.database
|
|
)
|
|
) as session:
|
|
return (
|
|
search_graph_with_session(
|
|
session,
|
|
query,
|
|
limit=limit,
|
|
max_entities=(
|
|
max_entities
|
|
),
|
|
)
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
parser = (
|
|
argparse.ArgumentParser()
|
|
)
|
|
|
|
parser.add_argument(
|
|
"query"
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=DEFAULT_LIMIT,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-entities",
|
|
type=int,
|
|
default=(
|
|
DEFAULT_MAX_ENTITIES
|
|
),
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
result = search_graph(
|
|
args.query,
|
|
limit=args.limit,
|
|
max_entities=(
|
|
args.max_entities
|
|
),
|
|
)
|
|
|
|
print(
|
|
json.dumps(
|
|
result,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|