2451 lines
71 KiB
Python
2451 lines
71 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
from typing import Any, Iterable
|
|
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
|
|
from .publication import public_query
|
|
from tools.retrieval.retrieval_contract import result_fields, source_references, error_result
|
|
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
|
from tools.retrieval.person_names import resolve_person_names, name_token_variants
|
|
|
|
GRAPH_SCOPE = "zpwiki"
|
|
DEFAULT_LIMIT = 40
|
|
DEFAULT_MAX_ENTITIES = 8
|
|
|
|
RELATIONSHIP_TYPES = (
|
|
"HAS_DOCUMENT|AUTHORED|IN_CATEGORY|HAS_KEYWORD|"
|
|
"DESCRIBES|HAS_WORK|EVIDENCED_BY|AUTHORED_DOCUMENT"
|
|
)
|
|
|
|
CATALOG_QUERY = """
|
|
MATCH (n)
|
|
WHERE n.graph_scope = $scope
|
|
AND (
|
|
n:Person
|
|
OR n:Thesis
|
|
OR n:Work
|
|
OR n:Keyword
|
|
OR n:Topic
|
|
OR n:Category
|
|
OR n:DocumentAuthor
|
|
)
|
|
RETURN
|
|
elementId(n) AS element_id,
|
|
CASE
|
|
WHEN n:Person THEN 'Person'
|
|
WHEN n:Thesis THEN 'Thesis'
|
|
WHEN n:Work THEN 'Work'
|
|
WHEN n:Keyword THEN 'Keyword'
|
|
WHEN n:Topic THEN 'Topic'
|
|
WHEN n:Category THEN 'Category'
|
|
WHEN n:DocumentAuthor THEN 'DocumentAuthor'
|
|
ELSE 'Other'
|
|
END AS kind,
|
|
n.id AS id,
|
|
n.name AS name,
|
|
n.title AS title,
|
|
n.heading AS heading,
|
|
n.start_year AS start_year,
|
|
n.year AS year,
|
|
n.thesis_type AS thesis_type,
|
|
n.thesis_type_label AS thesis_type_label,
|
|
n.work_type AS work_type,
|
|
n.work_type_label AS work_type_label,
|
|
n.person_kind AS person_kind,
|
|
n.source_document AS source_document
|
|
ORDER BY kind, name, title, heading
|
|
"""
|
|
|
|
NEIGHBORHOOD_QUERY = f"""
|
|
MATCH (start)
|
|
WHERE elementId(start) IN $start_ids
|
|
MATCH p=(start)-[:{RELATIONSHIP_TYPES}*1..2]-(target)
|
|
WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)
|
|
AND (
|
|
size($target_kinds) = 0
|
|
OR any(
|
|
node IN nodes(p)
|
|
WHERE
|
|
('Person' IN $target_kinds AND node:Person)
|
|
OR ('Thesis' IN $target_kinds AND node:Thesis)
|
|
OR ('Work' IN $target_kinds AND node:Work)
|
|
OR ('Keyword' IN $target_kinds AND node:Keyword)
|
|
OR ('Topic' IN $target_kinds AND node:Topic)
|
|
OR ('Category' IN $target_kinds AND node:Category)
|
|
OR (
|
|
'DocumentAuthor' IN $target_kinds
|
|
AND node:DocumentAuthor
|
|
)
|
|
OR ('Document' IN $target_kinds AND node:Document)
|
|
)
|
|
)
|
|
RETURN
|
|
[node IN nodes(p) | {{
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Thesis THEN 'Thesis'
|
|
WHEN node:Work THEN 'Work'
|
|
WHEN node:Document THEN 'Document'
|
|
WHEN node:Keyword THEN 'Keyword'
|
|
WHEN node:Topic THEN 'Topic'
|
|
WHEN node:Category THEN 'Category'
|
|
WHEN node:DocumentAuthor THEN 'DocumentAuthor'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
thesis_type: node.thesis_type,
|
|
thesis_type_label: node.thesis_type_label,
|
|
work_type: node.work_type,
|
|
work_type_label: node.work_type_label,
|
|
person_kind: node.person_kind,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY hops
|
|
LIMIT $limit
|
|
"""
|
|
|
|
PAIR_PATH_QUERY = f"""
|
|
MATCH (left), (right)
|
|
WHERE elementId(left) IN $start_ids
|
|
AND elementId(right) IN $start_ids
|
|
AND elementId(left) < elementId(right)
|
|
MATCH p=allShortestPaths(
|
|
(left)-[:{RELATIONSHIP_TYPES}*1..4]-(right)
|
|
)
|
|
WHERE all(node IN nodes(p) WHERE node.graph_scope = $scope)
|
|
RETURN
|
|
[node IN nodes(p) | {{
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Thesis THEN 'Thesis'
|
|
WHEN node:Work THEN 'Work'
|
|
WHEN node:Document THEN 'Document'
|
|
WHEN node:Keyword THEN 'Keyword'
|
|
WHEN node:Topic THEN 'Topic'
|
|
WHEN node:Category THEN 'Category'
|
|
WHEN node:DocumentAuthor THEN 'DocumentAuthor'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
thesis_type: node.thesis_type,
|
|
thesis_type_label: node.thesis_type_label,
|
|
work_type: node.work_type,
|
|
work_type_label: node.work_type_label,
|
|
person_kind: node.person_kind,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY hops
|
|
LIMIT $limit
|
|
"""
|
|
|
|
THESIS_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:AUTHORED]->
|
|
(thesis:Thesis)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND thesis.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($thesis_ids) = 0
|
|
OR elementId(thesis) IN $thesis_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR thesis.year IN $years
|
|
)
|
|
AND (
|
|
size($thesis_types) = 0
|
|
OR thesis.thesis_type IN $thesis_types
|
|
)
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
person.start_year AS start_year,
|
|
elementId(thesis) AS thesis_element_id,
|
|
thesis.id AS thesis_id,
|
|
thesis.title AS title,
|
|
thesis.heading AS heading,
|
|
thesis.year AS year,
|
|
thesis.thesis_type AS thesis_type,
|
|
thesis.thesis_type_label AS thesis_type_label,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url,
|
|
thesis.source_chunk_ids AS source_chunk_ids
|
|
ORDER BY person, year, thesis_type, title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
THESIS_PATHS_QUERY = """
|
|
MATCH p=
|
|
(person:Person)
|
|
-[:AUTHORED]->
|
|
(thesis:Thesis)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND thesis.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($thesis_ids) = 0
|
|
OR elementId(thesis) IN $thesis_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR thesis.year IN $years
|
|
)
|
|
AND (
|
|
size($thesis_types) = 0
|
|
OR thesis.thesis_type IN $thesis_types
|
|
)
|
|
RETURN
|
|
[node IN nodes(p) | {
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Thesis THEN 'Thesis'
|
|
WHEN node:Document THEN 'Document'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
thesis_type: node.thesis_type,
|
|
thesis_type_label: node.thesis_type_label,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY
|
|
person.name,
|
|
thesis.year,
|
|
thesis.thesis_type,
|
|
thesis.title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
WORK_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:HAS_WORK]->
|
|
(work:Work)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND work.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($work_ids) = 0
|
|
OR elementId(work) IN $work_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR work.year IN $years
|
|
)
|
|
AND (
|
|
size($work_types) = 0
|
|
OR work.work_type IN $work_types
|
|
)
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
person.start_year AS start_year,
|
|
elementId(work) AS work_element_id,
|
|
work.id AS work_id,
|
|
work.title AS title,
|
|
work.heading AS heading,
|
|
work.year AS year,
|
|
work.work_type AS work_type,
|
|
work.work_type_label AS work_type_label,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url,
|
|
work.source_chunk_ids AS source_chunk_ids
|
|
ORDER BY person, year, work_type, title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
WORK_PATHS_QUERY = """
|
|
MATCH p=
|
|
(person:Person)
|
|
-[:HAS_WORK]->
|
|
(work:Work)
|
|
-[:EVIDENCED_BY]->
|
|
(document:Document)
|
|
WHERE person.graph_scope = $scope
|
|
AND work.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($work_ids) = 0
|
|
OR elementId(work) IN $work_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR work.year IN $years
|
|
)
|
|
AND (
|
|
size($work_types) = 0
|
|
OR work.work_type IN $work_types
|
|
)
|
|
RETURN
|
|
[node IN nodes(p) | {
|
|
element_id: elementId(node),
|
|
kind: CASE
|
|
WHEN node:Person THEN 'Person'
|
|
WHEN node:Work THEN 'Work'
|
|
WHEN node:Document THEN 'Document'
|
|
ELSE 'Other'
|
|
END,
|
|
id: node.id,
|
|
name: node.name,
|
|
title: node.title,
|
|
heading: node.heading,
|
|
path: node.path,
|
|
start_year: node.start_year,
|
|
year: node.year,
|
|
work_type: node.work_type,
|
|
work_type_label: node.work_type_label,
|
|
source_document: node.source_document,
|
|
source_url: node.source_url
|
|
}] AS nodes,
|
|
[relationship IN relationships(p) |
|
|
type(relationship)
|
|
] AS relationships,
|
|
length(p) AS hops
|
|
ORDER BY
|
|
person.name,
|
|
work.year,
|
|
work.work_type,
|
|
work.title
|
|
LIMIT $limit
|
|
"""
|
|
|
|
START_YEAR_FACTS_QUERY = """
|
|
MATCH (person:Person)
|
|
WHERE person.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($years) = 0
|
|
OR person.start_year IN $years
|
|
)
|
|
OPTIONAL MATCH (document:Document)
|
|
WHERE document.graph_scope = $scope
|
|
AND document.path = person.source_document
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
person.start_year AS start_year,
|
|
person.source_document AS document_path,
|
|
document.source_url AS source_url
|
|
ORDER BY person
|
|
LIMIT $limit
|
|
"""
|
|
|
|
PERSON_KEYWORD_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(document:Document)
|
|
-[:HAS_KEYWORD]->
|
|
(keyword:Keyword)
|
|
WHERE person.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND keyword.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($keyword_ids) = 0
|
|
OR elementId(keyword) IN $keyword_ids
|
|
)
|
|
WITH
|
|
person,
|
|
keyword,
|
|
collect(DISTINCT document) AS documents
|
|
WITH
|
|
person,
|
|
keyword,
|
|
head(
|
|
[
|
|
document IN documents
|
|
WHERE document.path = person.source_document
|
|
]
|
|
+ documents
|
|
) AS document
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
elementId(keyword) AS keyword_element_id,
|
|
keyword.id AS keyword_id,
|
|
keyword.name AS keyword,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url
|
|
ORDER BY person, keyword
|
|
LIMIT $limit
|
|
"""
|
|
|
|
PERSON_CATEGORY_FACTS_QUERY = """
|
|
MATCH
|
|
(person:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(document:Document)
|
|
-[:IN_CATEGORY]->
|
|
(category:Category)
|
|
WHERE person.graph_scope = $scope
|
|
AND document.graph_scope = $scope
|
|
AND category.graph_scope = $scope
|
|
AND (
|
|
size($person_ids) = 0
|
|
OR elementId(person) IN $person_ids
|
|
)
|
|
AND (
|
|
size($category_ids) = 0
|
|
OR elementId(category) IN $category_ids
|
|
)
|
|
WITH
|
|
person,
|
|
category,
|
|
collect(DISTINCT document) AS documents
|
|
WITH
|
|
person,
|
|
category,
|
|
head(
|
|
[
|
|
document IN documents
|
|
WHERE document.path = person.source_document
|
|
]
|
|
+ documents
|
|
) AS document
|
|
RETURN
|
|
elementId(person) AS person_element_id,
|
|
person.name AS person,
|
|
elementId(category) AS category_element_id,
|
|
category.name AS category,
|
|
category.category_type AS category_type,
|
|
category.year AS category_year,
|
|
document.path AS document_path,
|
|
document.source_url AS source_url
|
|
ORDER BY person, category
|
|
LIMIT $limit
|
|
"""
|
|
|
|
SHARED_KEYWORD_FACTS_QUERY = """
|
|
MATCH
|
|
(left:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(left_document:Document)
|
|
-[:HAS_KEYWORD]->
|
|
(keyword:Keyword)
|
|
MATCH
|
|
(right:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(right_document:Document)
|
|
-[:HAS_KEYWORD]->
|
|
(keyword)
|
|
WHERE left.graph_scope = $scope
|
|
AND right.graph_scope = $scope
|
|
AND left_document.graph_scope = $scope
|
|
AND right_document.graph_scope = $scope
|
|
AND keyword.graph_scope = $scope
|
|
AND elementId(left) IN $person_ids
|
|
AND elementId(right) IN $person_ids
|
|
AND elementId(left) < elementId(right)
|
|
WITH
|
|
left,
|
|
right,
|
|
keyword,
|
|
collect(DISTINCT left_document) AS left_documents,
|
|
collect(DISTINCT right_document) AS right_documents
|
|
WITH
|
|
left,
|
|
right,
|
|
keyword,
|
|
head(
|
|
[
|
|
document IN left_documents
|
|
WHERE document.path = left.source_document
|
|
]
|
|
+ left_documents
|
|
) AS left_document,
|
|
head(
|
|
[
|
|
document IN right_documents
|
|
WHERE document.path = right.source_document
|
|
]
|
|
+ right_documents
|
|
) AS right_document
|
|
RETURN
|
|
elementId(left) AS left_person_element_id,
|
|
left.name AS left_person,
|
|
left_document.path AS left_document_path,
|
|
left_document.source_url AS left_source_url,
|
|
elementId(right) AS right_person_element_id,
|
|
right.name AS right_person,
|
|
right_document.path AS right_document_path,
|
|
right_document.source_url AS right_source_url,
|
|
elementId(keyword) AS keyword_element_id,
|
|
keyword.id AS keyword_id,
|
|
keyword.name AS keyword
|
|
ORDER BY left_person, right_person, keyword
|
|
LIMIT $limit
|
|
"""
|
|
|
|
SHARED_CATEGORY_FACTS_QUERY = """
|
|
MATCH
|
|
(left:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(left_document:Document)
|
|
-[:IN_CATEGORY]->
|
|
(category:Category)
|
|
MATCH
|
|
(right:Person)
|
|
-[:HAS_DOCUMENT]->
|
|
(right_document:Document)
|
|
-[:IN_CATEGORY]->
|
|
(category)
|
|
WHERE left.graph_scope = $scope
|
|
AND right.graph_scope = $scope
|
|
AND left_document.graph_scope = $scope
|
|
AND right_document.graph_scope = $scope
|
|
AND category.graph_scope = $scope
|
|
AND elementId(left) IN $person_ids
|
|
AND elementId(right) IN $person_ids
|
|
AND elementId(left) < elementId(right)
|
|
WITH
|
|
left,
|
|
right,
|
|
category,
|
|
collect(DISTINCT left_document) AS left_documents,
|
|
collect(DISTINCT right_document) AS right_documents
|
|
WITH
|
|
left,
|
|
right,
|
|
category,
|
|
head(
|
|
[
|
|
document IN left_documents
|
|
WHERE document.path = left.source_document
|
|
]
|
|
+ left_documents
|
|
) AS left_document,
|
|
head(
|
|
[
|
|
document IN right_documents
|
|
WHERE document.path = right.source_document
|
|
]
|
|
+ right_documents
|
|
) AS right_document
|
|
RETURN
|
|
elementId(left) AS left_person_element_id,
|
|
left.name AS left_person,
|
|
left_document.path AS left_document_path,
|
|
left_document.source_url AS left_source_url,
|
|
elementId(right) AS right_person_element_id,
|
|
right.name AS right_person,
|
|
right_document.path AS right_document_path,
|
|
right_document.source_url AS right_source_url,
|
|
elementId(category) AS category_element_id,
|
|
category.name AS category,
|
|
category.category_type AS category_type,
|
|
category.year AS category_year
|
|
ORDER BY left_person, right_person, category
|
|
LIMIT $limit
|
|
"""
|
|
|
|
SHARED_DOCUMENT_AUTHOR_FACTS_QUERY = """
|
|
MATCH
|
|
(author:DocumentAuthor)
|
|
-[:AUTHORED_DOCUMENT]->
|
|
(left_document:Document)
|
|
<-[:HAS_DOCUMENT]-
|
|
(left:Person)
|
|
MATCH
|
|
(author)
|
|
-[:AUTHORED_DOCUMENT]->
|
|
(right_document:Document)
|
|
<-[:HAS_DOCUMENT]-
|
|
(right:Person)
|
|
WHERE author.graph_scope = $scope
|
|
AND left.graph_scope = $scope
|
|
AND right.graph_scope = $scope
|
|
AND left_document.graph_scope = $scope
|
|
AND right_document.graph_scope = $scope
|
|
AND elementId(left) IN $person_ids
|
|
AND elementId(right) IN $person_ids
|
|
AND elementId(left) < elementId(right)
|
|
WITH
|
|
left,
|
|
right,
|
|
author,
|
|
collect(DISTINCT left_document) AS left_documents,
|
|
collect(DISTINCT right_document) AS right_documents
|
|
WITH
|
|
left,
|
|
right,
|
|
author,
|
|
head(
|
|
[
|
|
document IN left_documents
|
|
WHERE document.path = left.source_document
|
|
]
|
|
+ left_documents
|
|
) AS left_document,
|
|
head(
|
|
[
|
|
document IN right_documents
|
|
WHERE document.path = right.source_document
|
|
]
|
|
+ right_documents
|
|
) AS right_document
|
|
RETURN
|
|
elementId(left) AS left_person_element_id,
|
|
left.name AS left_person,
|
|
left_document.path AS left_document_path,
|
|
left_document.source_url AS left_source_url,
|
|
elementId(right) AS right_person_element_id,
|
|
right.name AS right_person,
|
|
right_document.path AS right_document_path,
|
|
right_document.source_url AS right_source_url,
|
|
elementId(author) AS document_author_element_id,
|
|
author.id AS document_author_id,
|
|
author.name AS document_author
|
|
ORDER BY left_person, right_person, document_author
|
|
LIMIT $limit
|
|
"""
|
|
|
|
|
|
def normalize_for_match(
|
|
value: str,
|
|
) -> str:
|
|
decomposed = unicodedata.normalize(
|
|
"NFKD",
|
|
str(value),
|
|
)
|
|
|
|
without_marks = "".join(
|
|
character for character in decomposed if not unicodedata.combining(character)
|
|
)
|
|
|
|
normalized = re.sub(
|
|
r"[^a-zA-Z0-9]+",
|
|
" ",
|
|
without_marks.casefold(),
|
|
)
|
|
|
|
return re.sub(
|
|
r"\s+",
|
|
" ",
|
|
normalized,
|
|
).strip()
|
|
|
|
|
|
def normalized_tokens(
|
|
value: str,
|
|
) -> list[str]:
|
|
normalized = normalize_for_match(value)
|
|
|
|
if not normalized:
|
|
return []
|
|
|
|
return normalized.split()
|
|
|
|
|
|
def has_stem(
|
|
tokens: Iterable[str],
|
|
stem: str,
|
|
) -> bool:
|
|
return any(token.startswith(stem) for token in tokens)
|
|
|
|
|
|
def explicit_keyword_phrase(query: str) -> str | None:
|
|
"""Identify a requested tag without treating 'ZP Wiki' as the tag ZP."""
|
|
quoted = re.search(
|
|
r"\b(?:kľúčové\s+slovo|kľúčovým\s+slovom|kľúčového\s+slova|"
|
|
r"klucove\s+slovo|klucovym\s+slovom|klucoveho\s+slova|tagom|tagu|tag)"
|
|
r'\s+[„“"«]([^„“”"»]+)[“”"»]',
|
|
query,
|
|
flags=re.I,
|
|
)
|
|
if quoted:
|
|
return normalize_for_match(quoted[1])
|
|
match = re.search(
|
|
r"\b(?:klucove\s+slovo|klucovym\s+slovom|klucoveho\s+slova|" r"tagom|tagu|tag)\s+(.+)",
|
|
normalize_for_match(query),
|
|
)
|
|
if not match:
|
|
return None
|
|
phrase = re.split(
|
|
r"\s+(?:pri\s+osob\w*|u\s+osob\w*|a\s+(?:zaroven|sucasne|diplomov\w*|bakalarsk\w*|"
|
|
r"dizertacn\w*|timov\w*|vyskumn\w*|projekt\w*|prac\w*|jej|jeho|ich|rok|pomenuj|uved\w*)|"
|
|
r"zaroven\b|sucasne\b|(?:v|z|pre)\s+rok\w*\b|ak\s+|uved\w*\s+|"
|
|
r"pomenuj\s+|maju\s+|ma\s+)",
|
|
match.group(1),
|
|
maxsplit=1,
|
|
)[0].strip()
|
|
return phrase or None
|
|
|
|
|
|
def extract_query_constraints(
|
|
query: str,
|
|
) -> dict[str, Any]:
|
|
normalized = normalize_for_match(query)
|
|
|
|
tokens = normalized.split()
|
|
token_set = set(tokens)
|
|
|
|
years = sorted(
|
|
{
|
|
int(value)
|
|
for value in re.findall(
|
|
r"\b(\d{4})\b",
|
|
normalized,
|
|
)
|
|
}
|
|
)
|
|
|
|
has_work_word = has_stem(
|
|
tokens,
|
|
"prac",
|
|
)
|
|
|
|
has_project_word = has_stem(
|
|
tokens,
|
|
"projekt",
|
|
)
|
|
|
|
thesis_types: list[str] = []
|
|
work_types: list[str] = []
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"bakalar",
|
|
)
|
|
and has_work_word
|
|
) or "bp" in token_set:
|
|
thesis_types.append("bachelor_thesis")
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"diplom",
|
|
)
|
|
and has_work_word
|
|
) or "dp" in token_set:
|
|
thesis_types.append("diploma_thesis")
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"dizert",
|
|
)
|
|
and has_work_word
|
|
) or "phd" in token_set:
|
|
thesis_types.append("doctoral_thesis")
|
|
|
|
if has_stem(tokens, "bakalar") and has_project_word:
|
|
work_types.append("bachelor_project")
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"diplom",
|
|
)
|
|
and has_project_word
|
|
):
|
|
work_types.append("diploma_project")
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"timov",
|
|
)
|
|
and has_project_word
|
|
):
|
|
work_types.append("team_project")
|
|
|
|
if (
|
|
has_stem(
|
|
tokens,
|
|
"vyskum",
|
|
)
|
|
and has_project_word
|
|
):
|
|
work_types.append("research_project")
|
|
|
|
if "tp" in token_set:
|
|
work_types.append("team_project")
|
|
|
|
if "vp" in token_set:
|
|
work_types.append("research_project")
|
|
|
|
thesis_types = list(dict.fromkeys(thesis_types))
|
|
|
|
work_types = list(dict.fromkeys(work_types))
|
|
|
|
mentions_start_year = has_stem(
|
|
tokens,
|
|
"zaciat",
|
|
) and has_stem(
|
|
tokens,
|
|
"stud",
|
|
)
|
|
|
|
mentions_keyword = (
|
|
has_stem(
|
|
tokens,
|
|
"kluc",
|
|
)
|
|
or "tag" in token_set
|
|
or "tagy" in token_set
|
|
)
|
|
|
|
mentions_topic = has_stem(
|
|
tokens,
|
|
"tem",
|
|
)
|
|
|
|
mentions_category = has_stem(
|
|
tokens,
|
|
"kategori",
|
|
)
|
|
|
|
mentions_document_author = has_stem(
|
|
tokens,
|
|
"autor",
|
|
) and (
|
|
has_stem(
|
|
tokens,
|
|
"dokument",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"strank",
|
|
)
|
|
or "wiki" in token_set
|
|
)
|
|
|
|
mentions_title = (
|
|
has_stem(tokens, "nazov")
|
|
or has_stem(tokens, "nazv")
|
|
or (has_work_word and "vola" in token_set)
|
|
)
|
|
mentions_academic_year = (
|
|
has_work_word
|
|
and bool(token_set & {"rok", "roka", "roku", "rokom", "roky", "rokoch"})
|
|
and not mentions_start_year
|
|
)
|
|
|
|
mentions_shared = (
|
|
has_stem(
|
|
tokens,
|
|
"spoloc",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"rovnak",
|
|
)
|
|
or (mentions_academic_year and has_stem(tokens, "spaj"))
|
|
or (
|
|
mentions_academic_year
|
|
and bool(token_set & {"obaja", "obidvaja", "obe", "obidve", "vsetci"})
|
|
)
|
|
)
|
|
|
|
mentions_person_answer = "kto" in token_set or (
|
|
has_stem(
|
|
tokens,
|
|
"ktor",
|
|
)
|
|
and (
|
|
has_stem(
|
|
tokens,
|
|
"student",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"osob",
|
|
)
|
|
or has_stem(
|
|
tokens,
|
|
"clov",
|
|
)
|
|
)
|
|
)
|
|
|
|
# Asking *which works* requests records, even without the word "title".
|
|
# Keep this separate from title-only lookup: untitled projects must survive.
|
|
academic_noun = r"(?:prac(?:a|e|u)?|projekt(?:y|ov|u)?)"
|
|
modifiers = (
|
|
r"(?:(?:vsetk\w*|(?:akademick|zaverecn|bakalarsk|diplomov|timov|vyskumn|dizertacn)\w*)\s+)*"
|
|
)
|
|
mentions_work_listing = (
|
|
bool(
|
|
re.search(
|
|
rf"\b(?:ake|aku|aky|ktore|ktoru|ktory|vypis|vymenuj|uved|zobraz|ukaz|zoznam|prehlad)\s+{modifiers}{academic_noun}\b",
|
|
normalized,
|
|
)
|
|
or re.search(
|
|
r"\b(?:ake|ktore)\s+(?:ma|mal|maju|mali)\b.*?\b(?:prace|projekty)\b", normalized
|
|
)
|
|
)
|
|
and not bool(token_set & {"kolko", "pocet", "spocitaj"})
|
|
and not mentions_person_answer
|
|
)
|
|
|
|
mentions_thesis = bool(thesis_types) or (
|
|
has_stem(
|
|
tokens,
|
|
"zaverec",
|
|
)
|
|
and has_work_word
|
|
)
|
|
|
|
mentions_work = bool(work_types) or (
|
|
mentions_work_listing and has_project_word and not has_work_word
|
|
)
|
|
|
|
target_kinds: list[str] = []
|
|
|
|
if mentions_keyword:
|
|
target_kinds.append("Keyword")
|
|
|
|
if mentions_topic:
|
|
target_kinds.extend(
|
|
[
|
|
"Keyword",
|
|
"Topic",
|
|
]
|
|
)
|
|
|
|
if mentions_category:
|
|
target_kinds.append("Category")
|
|
|
|
if mentions_document_author:
|
|
target_kinds.append("DocumentAuthor")
|
|
|
|
if mentions_thesis:
|
|
target_kinds.append("Thesis")
|
|
|
|
if mentions_work:
|
|
target_kinds.append("Work")
|
|
|
|
if (mentions_title or mentions_work_listing) and not mentions_thesis and not mentions_work:
|
|
target_kinds.extend(
|
|
[
|
|
"Thesis",
|
|
"Work",
|
|
]
|
|
)
|
|
|
|
if years and not mentions_start_year and not mentions_thesis and not mentions_work:
|
|
target_kinds.extend(
|
|
[
|
|
"Thesis",
|
|
"Work",
|
|
]
|
|
)
|
|
|
|
if mentions_person_answer and not mentions_shared:
|
|
target_kinds = ["Person"]
|
|
|
|
return {
|
|
"years": years,
|
|
"thesis_types": (thesis_types),
|
|
"work_types": (work_types),
|
|
"mentions_start_year": (mentions_start_year),
|
|
"mentions_keyword": (mentions_keyword),
|
|
"mentions_topic": (mentions_topic),
|
|
"mentions_category": (mentions_category),
|
|
"mentions_document_author": (mentions_document_author),
|
|
"mentions_title": (mentions_title),
|
|
"mentions_academic_year": mentions_academic_year,
|
|
"mentions_work_listing": mentions_work_listing,
|
|
"keyword_phrase": explicit_keyword_phrase(query),
|
|
"mentions_count": bool(token_set & {"kolko", "pocet", "spocitaj"}),
|
|
"mentions_shared": (mentions_shared),
|
|
"mentions_person_answer": (mentions_person_answer),
|
|
"mentions_thesis": (mentions_thesis),
|
|
"mentions_work": (mentions_work),
|
|
"target_kinds": list(dict.fromkeys(target_kinds)),
|
|
}
|
|
|
|
|
|
def entity_aliases(
|
|
entity: dict[str, Any],
|
|
) -> list[str]:
|
|
kind = str(entity.get("kind") or "")
|
|
|
|
values: list[Any] = []
|
|
|
|
if kind in {
|
|
"Person",
|
|
"DocumentAuthor",
|
|
}:
|
|
values.append(entity.get("name"))
|
|
|
|
elif kind in {
|
|
"Keyword",
|
|
"Topic",
|
|
}:
|
|
values.extend(
|
|
[
|
|
entity.get("name"),
|
|
entity.get("id"),
|
|
]
|
|
)
|
|
|
|
elif kind == "Category":
|
|
values.append(entity.get("name"))
|
|
|
|
elif kind in {
|
|
"Thesis",
|
|
"Work",
|
|
}:
|
|
values.append(entity.get("title"))
|
|
|
|
aliases: list[str] = []
|
|
|
|
for value in values:
|
|
if value is None:
|
|
continue
|
|
|
|
normalized = normalize_for_match(str(value))
|
|
|
|
if normalized:
|
|
aliases.append(normalized)
|
|
|
|
return list(dict.fromkeys(aliases))
|
|
|
|
|
|
def person_name_match(query: str, alias: str) -> tuple[float, tuple[int, int] | None]:
|
|
"""Match the complete name in one contiguous window, preserving token order."""
|
|
words, name = normalize_for_match(query).split(), normalize_for_match(alias).split()
|
|
if len(name) < 2:
|
|
return 0.0, None
|
|
best = (0.0, None)
|
|
for start in range(len(words) - len(name) + 1):
|
|
window = words[start : start + len(name)]
|
|
if window == name:
|
|
return 200.0 + len(name), (start, start + len(name))
|
|
if all(
|
|
observed in name_token_variants(canonical) for canonical, observed in zip(name, window)
|
|
):
|
|
best = (180.0 + len(name), (start, start + len(name)))
|
|
return best
|
|
|
|
|
|
def named_author_phrases(query: str) -> list[str]:
|
|
normalized = normalize_for_match(query.replace(",", " separator "))
|
|
match = re.search(r"\bautor(?:a|om)?(?:\s+dokument(?:u|ov))?\s+(.+)", normalized)
|
|
if not match:
|
|
return []
|
|
name = re.split(r"\s+(?:v|s|so|a|ak|ktory\w*|pri|spojen\w*)\b", match.group(1), maxsplit=1)[0]
|
|
return (
|
|
[name]
|
|
if 2 <= len(name.split()) <= 4
|
|
and name.split()[0]
|
|
not in {
|
|
"dokumentu",
|
|
"dokumentov",
|
|
"je",
|
|
"sa",
|
|
"pri",
|
|
"osoby",
|
|
"osobe",
|
|
"osobu",
|
|
"osob",
|
|
"ktory",
|
|
"ktora",
|
|
"ktoreho",
|
|
"studentov",
|
|
"studenta",
|
|
}
|
|
else []
|
|
)
|
|
|
|
|
|
def explicit_keyword_entities(phrase: str, catalog: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Resolve a requested tag as a complete leading phrase, not unrelated words."""
|
|
normalized = normalize_for_match(phrase)
|
|
matches = []
|
|
for entity in catalog:
|
|
if entity.get("kind") != "Keyword":
|
|
continue
|
|
aliases = entity_aliases(entity)
|
|
matching = [alias for alias in aliases if normalized == alias]
|
|
if matching:
|
|
best = max(matching, key=len)
|
|
matches.append({**entity, "matched_alias": best, "match_score": 200.0 + len(best)})
|
|
if not matches:
|
|
return []
|
|
# Prefer the longest full tag when one tag is a prefix of another.
|
|
longest = max(len(row["matched_alias"]) for row in matches)
|
|
return [row for row in matches if len(row["matched_alias"]) == longest]
|
|
|
|
|
|
def tokens_match(
|
|
left: str,
|
|
right: str,
|
|
) -> bool:
|
|
if left == right:
|
|
return True
|
|
|
|
shorter = min(
|
|
len(left),
|
|
len(right),
|
|
)
|
|
|
|
if shorter <= 2:
|
|
return False
|
|
|
|
required = 3 if shorter <= 4 else 4 if shorter <= 6 else 5
|
|
|
|
return left[:required] == right[:required]
|
|
|
|
|
|
def alias_match_score(
|
|
normalized_query: str,
|
|
query_tokens: set[str],
|
|
alias: str,
|
|
) -> float:
|
|
alias_tokens = alias.split()
|
|
|
|
if not alias_tokens:
|
|
return 0.0
|
|
|
|
if len(alias_tokens) == 1:
|
|
if alias not in query_tokens:
|
|
return 0.0
|
|
|
|
return (
|
|
50.0
|
|
+ min(
|
|
len(alias),
|
|
20,
|
|
)
|
|
/ 100.0
|
|
)
|
|
|
|
padded_query = f" {normalized_query} "
|
|
|
|
padded_alias = f" {alias} "
|
|
|
|
if padded_alias in padded_query:
|
|
score = (
|
|
100.0
|
|
+ len(alias_tokens) * 5.0
|
|
+ min(
|
|
len(alias),
|
|
100,
|
|
)
|
|
/ 100.0
|
|
)
|
|
|
|
if normalized_query == alias:
|
|
score += 50.0
|
|
|
|
return score
|
|
|
|
if all(
|
|
any(
|
|
tokens_match(
|
|
alias_token,
|
|
query_token,
|
|
)
|
|
for query_token in query_tokens
|
|
)
|
|
for alias_token in alias_tokens
|
|
):
|
|
return (
|
|
95.0
|
|
+ len(alias_tokens) * 5.0
|
|
+ min(
|
|
len(alias),
|
|
100,
|
|
)
|
|
/ 100.0
|
|
)
|
|
|
|
return 0.0
|
|
|
|
|
|
def suppress_nested_entity_matches(
|
|
matches: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
result: list[dict[str, Any]] = []
|
|
|
|
for item in matches:
|
|
kind = str(item.get("kind") or "")
|
|
|
|
if kind not in {
|
|
"Thesis",
|
|
"Work",
|
|
}:
|
|
result.append(item)
|
|
continue
|
|
|
|
alias = str(item.get("matched_alias") or "")
|
|
|
|
score = float(item.get("match_score") or 0.0)
|
|
|
|
dominated = False
|
|
|
|
for other in matches:
|
|
if other is item:
|
|
continue
|
|
|
|
if other.get("kind") != kind:
|
|
continue
|
|
|
|
other_alias = str(other.get("matched_alias") or "")
|
|
|
|
other_score = float(other.get("match_score") or 0.0)
|
|
|
|
if not alias or not other_alias or alias == other_alias:
|
|
continue
|
|
|
|
if alias in other_alias and len(other_alias) > len(alias) and other_score > score:
|
|
dominated = True
|
|
break
|
|
|
|
if not dominated:
|
|
result.append(item)
|
|
|
|
return result
|
|
|
|
|
|
def resolve_entities(
|
|
query: str,
|
|
catalog: list[dict[str, Any]],
|
|
*,
|
|
max_entities: int = (DEFAULT_MAX_ENTITIES),
|
|
preferred_kinds: Iterable[str] = (),
|
|
) -> list[dict[str, Any]]:
|
|
normalized_query = normalize_for_match(query)
|
|
|
|
query_tokens = set(normalized_query.split())
|
|
|
|
preferred = set(preferred_kinds)
|
|
|
|
matches: list[dict[str, Any]] = []
|
|
|
|
for entity in catalog:
|
|
best_score = 0.0
|
|
best_alias = ""
|
|
|
|
for alias in entity_aliases(entity):
|
|
name_span = None
|
|
if entity.get("kind") in {"Person", "DocumentAuthor"}:
|
|
score, name_span = person_name_match(normalized_query, alias)
|
|
else:
|
|
score = alias_match_score(
|
|
normalized_query,
|
|
query_tokens,
|
|
alias,
|
|
)
|
|
|
|
if score > 0 and entity.get("kind") in preferred:
|
|
score += 2.0
|
|
|
|
if score > best_score:
|
|
best_score = score
|
|
best_alias = alias
|
|
best_name_span = name_span
|
|
|
|
if best_score <= 0.0:
|
|
continue
|
|
|
|
item = dict(entity)
|
|
|
|
item["matched_alias"] = best_alias
|
|
if entity.get("kind") in {"Person", "DocumentAuthor"}:
|
|
item["matched_name_span"] = best_name_span
|
|
|
|
item["match_score"] = round(
|
|
best_score,
|
|
3,
|
|
)
|
|
|
|
matches.append(item)
|
|
|
|
exact_name_spans = {
|
|
tuple(item["matched_name_span"])
|
|
for item in matches
|
|
if item.get("matched_name_span") is not None and item["match_score"] >= 200
|
|
}
|
|
matches = [
|
|
item
|
|
for item in matches
|
|
if not (
|
|
item.get("matched_name_span") is not None
|
|
and tuple(item["matched_name_span"]) in exact_name_spans
|
|
and item["match_score"] < 200
|
|
)
|
|
]
|
|
|
|
person_aliases = {
|
|
str(item.get("matched_alias") or "") for item in matches if item.get("kind") == "Person"
|
|
}
|
|
|
|
matches = [
|
|
item
|
|
for item in matches
|
|
if not (
|
|
item.get("kind") == "DocumentAuthor"
|
|
and str(item.get("matched_alias") or "") in person_aliases
|
|
)
|
|
]
|
|
|
|
matches = suppress_nested_entity_matches(matches)
|
|
|
|
matches.sort(
|
|
key=lambda item: (
|
|
-float(item["match_score"]),
|
|
str(item.get("kind") or ""),
|
|
str(item.get("name") or item.get("title") or item.get("id") or ""),
|
|
)
|
|
)
|
|
|
|
selected: list[dict[str, Any]] = []
|
|
|
|
seen_ids: set[str] = set()
|
|
|
|
for item in matches:
|
|
element_id = str(item.get("element_id") or "")
|
|
|
|
if not element_id:
|
|
continue
|
|
|
|
if element_id in seen_ids:
|
|
continue
|
|
|
|
seen_ids.add(element_id)
|
|
|
|
selected.append(item)
|
|
|
|
if len(selected) >= max_entities:
|
|
break
|
|
|
|
return selected
|
|
|
|
|
|
def rows_from_result(
|
|
result: Any,
|
|
) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
|
|
for record in result:
|
|
if isinstance(
|
|
record,
|
|
dict,
|
|
):
|
|
rows.append(dict(record))
|
|
continue
|
|
|
|
data = getattr(
|
|
record,
|
|
"data",
|
|
None,
|
|
)
|
|
|
|
if callable(data):
|
|
rows.append(dict(data()))
|
|
continue
|
|
|
|
rows.append(dict(record))
|
|
|
|
return rows
|
|
|
|
|
|
def run_rows(
|
|
session: Any,
|
|
query: str,
|
|
**parameters: Any,
|
|
) -> list[dict[str, Any]]:
|
|
return rows_from_result(
|
|
session.run(
|
|
query,
|
|
**parameters,
|
|
)
|
|
)
|
|
|
|
|
|
def deduplicate_paths(
|
|
paths: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
result: list[dict[str, Any]] = []
|
|
|
|
seen: set[str] = set()
|
|
|
|
for path in paths:
|
|
key = json.dumps(
|
|
{
|
|
"nodes": path.get(
|
|
"nodes",
|
|
[],
|
|
),
|
|
"relationships": (
|
|
path.get(
|
|
"relationships",
|
|
[],
|
|
)
|
|
),
|
|
},
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
default=str,
|
|
)
|
|
|
|
if key in seen:
|
|
continue
|
|
|
|
seen.add(key)
|
|
|
|
result.append(path)
|
|
|
|
return result
|
|
|
|
|
|
def deduplicate_rows(
|
|
rows: list[dict[str, Any]],
|
|
key_fields: tuple[
|
|
str,
|
|
...,
|
|
],
|
|
) -> list[dict[str, Any]]:
|
|
result: list[dict[str, Any]] = []
|
|
|
|
seen: set[tuple[str, ...]] = set()
|
|
|
|
for row in rows:
|
|
key = tuple(str(row.get(field) or "") for field in key_fields)
|
|
|
|
if key in seen:
|
|
continue
|
|
|
|
seen.add(key)
|
|
|
|
result.append(row)
|
|
|
|
return result
|
|
|
|
|
|
def collect_documents(
|
|
resolved_entities: list[dict[str, Any]],
|
|
paths: list[dict[str, Any]],
|
|
facts: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
documents: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
def add_document(
|
|
path: Any,
|
|
source_url: Any = None,
|
|
) -> None:
|
|
value = str(path or "").strip()
|
|
|
|
if not value:
|
|
return
|
|
|
|
current = documents.setdefault(
|
|
value,
|
|
{
|
|
"document_path": (value),
|
|
"source_url": None,
|
|
},
|
|
)
|
|
|
|
if source_url and not current.get("source_url"):
|
|
current["source_url"] = str(source_url)
|
|
|
|
for entity in resolved_entities:
|
|
add_document(entity.get("source_document"))
|
|
|
|
for path in paths:
|
|
for node in path.get(
|
|
"nodes",
|
|
[],
|
|
):
|
|
if not isinstance(
|
|
node,
|
|
dict,
|
|
):
|
|
continue
|
|
|
|
if node.get("kind") == "Document":
|
|
add_document(
|
|
node.get("path"),
|
|
node.get("source_url"),
|
|
)
|
|
|
|
add_document(node.get("source_document"))
|
|
|
|
for fact in facts:
|
|
add_document(
|
|
fact.get("document_path"),
|
|
fact.get("source_url"),
|
|
)
|
|
|
|
add_document(
|
|
fact.get("left_document_path"),
|
|
fact.get("left_source_url"),
|
|
)
|
|
|
|
add_document(
|
|
fact.get("right_document_path"),
|
|
fact.get("right_source_url"),
|
|
)
|
|
|
|
return sorted(
|
|
documents.values(),
|
|
key=lambda item: (item["document_path"]),
|
|
)
|
|
|
|
|
|
def build_facts(
|
|
thesis_rows: list[dict[str, Any]],
|
|
work_rows: list[dict[str, Any]],
|
|
start_year_rows: list[dict[str, Any]],
|
|
person_keyword_rows: list[dict[str, Any]],
|
|
person_category_rows: list[dict[str, Any]],
|
|
shared_keyword_rows: list[dict[str, Any]],
|
|
shared_category_rows: list[dict[str, Any]],
|
|
shared_document_author_rows: list[dict[str, Any]],
|
|
*,
|
|
mentions_shared: bool,
|
|
) -> list[dict[str, Any]]:
|
|
facts: list[dict[str, Any]] = []
|
|
|
|
for row in thesis_rows:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "person_thesis"
|
|
|
|
facts.append(fact)
|
|
|
|
for row in work_rows:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "person_work"
|
|
|
|
facts.append(fact)
|
|
|
|
unique_start_rows = deduplicate_rows(
|
|
start_year_rows,
|
|
("person_element_id",),
|
|
)
|
|
|
|
for row in unique_start_rows:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "person_start_year"
|
|
|
|
facts.append(fact)
|
|
|
|
unique_person_keywords = deduplicate_rows(
|
|
person_keyword_rows,
|
|
(
|
|
"person_element_id",
|
|
"keyword_element_id",
|
|
),
|
|
)
|
|
|
|
for row in unique_person_keywords:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "person_keyword"
|
|
|
|
facts.append(fact)
|
|
|
|
unique_person_categories = deduplicate_rows(
|
|
person_category_rows,
|
|
(
|
|
"person_element_id",
|
|
"category_element_id",
|
|
),
|
|
)
|
|
|
|
for row in unique_person_categories:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "person_category"
|
|
|
|
facts.append(fact)
|
|
|
|
unique_shared_keywords = deduplicate_rows(
|
|
shared_keyword_rows,
|
|
(
|
|
"left_person_element_id",
|
|
"right_person_element_id",
|
|
"keyword_element_id",
|
|
),
|
|
)
|
|
|
|
for row in unique_shared_keywords:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "shared_keyword"
|
|
|
|
facts.append(fact)
|
|
|
|
unique_shared_categories = deduplicate_rows(
|
|
shared_category_rows,
|
|
(
|
|
"left_person_element_id",
|
|
"right_person_element_id",
|
|
"category_element_id",
|
|
),
|
|
)
|
|
|
|
for row in unique_shared_categories:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "shared_category"
|
|
|
|
facts.append(fact)
|
|
|
|
unique_shared_authors = deduplicate_rows(
|
|
shared_document_author_rows,
|
|
(
|
|
"left_person_element_id",
|
|
"right_person_element_id",
|
|
"document_author_element_id",
|
|
),
|
|
)
|
|
|
|
for row in unique_shared_authors:
|
|
fact = dict(row)
|
|
|
|
fact["fact_type"] = "shared_document_author"
|
|
|
|
facts.append(fact)
|
|
|
|
if mentions_shared and len(unique_start_rows) >= 2:
|
|
ordered = sorted(
|
|
unique_start_rows,
|
|
key=lambda row: str(row.get("person") or ""),
|
|
)
|
|
|
|
years = [row.get("start_year") for row in ordered]
|
|
|
|
known_years = [year for year in years if year is not None]
|
|
|
|
same = len(known_years) == len(years) and len(set(known_years)) == 1
|
|
|
|
facts.append(
|
|
{
|
|
"fact_type": ("start_year_comparison"),
|
|
"people": [str(row.get("person") or "") for row in ordered],
|
|
"start_years": {
|
|
str(row.get("person") or ""): row.get("start_year") for row in ordered
|
|
},
|
|
"same_start_year": (same),
|
|
"shared_start_year": (known_years[0] if same else None),
|
|
}
|
|
)
|
|
|
|
return facts
|
|
|
|
|
|
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY = """
|
|
MATCH (person:Person)-[:HAS_DOCUMENT]->(document:Document)
|
|
MATCH (author:DocumentAuthor)-[:AUTHORED_DOCUMENT]->(document)
|
|
WHERE person.graph_scope = $scope AND document.graph_scope = $scope
|
|
AND author.graph_scope = $scope
|
|
AND (size($person_ids) = 0 OR elementId(person) IN $person_ids)
|
|
AND (size($author_ids) = 0 OR elementId(author) IN $author_ids)
|
|
OPTIONAL MATCH (document)-[:IN_CATEGORY]->(category:Category)
|
|
WHERE category.graph_scope = $scope
|
|
WITH person, document, author, category
|
|
WHERE size($category_ids) = 0 OR elementId(category) IN $category_ids
|
|
RETURN DISTINCT elementId(person) AS person_element_id, person.name AS person,
|
|
elementId(author) AS document_author_element_id, author.name AS document_author,
|
|
elementId(category) AS category_element_id, category.name AS category,
|
|
document.path AS document_path, document.source_url AS source_url
|
|
ORDER BY person, document_author, document_path
|
|
LIMIT $limit
|
|
"""
|
|
|
|
DOCUMENT_SOURCES_QUERY = """
|
|
MATCH (document:Document)
|
|
WHERE document.graph_scope = $scope AND document.path IN $paths
|
|
RETURN document.path AS document_path, document.source_url AS source_url, document.title AS title
|
|
ORDER BY document_path
|
|
"""
|
|
|
|
# Restrict the complete retrieval pipeline, including vocabulary and traversal.
|
|
CATALOG_QUERY = public_query(CATALOG_QUERY, nodes=("n",))
|
|
NEIGHBORHOOD_QUERY = public_query(NEIGHBORHOOD_QUERY, path=True)
|
|
PAIR_PATH_QUERY = public_query(PAIR_PATH_QUERY, path=True)
|
|
THESIS_FACTS_QUERY = public_query(THESIS_FACTS_QUERY, nodes=("person",), documents=("document",))
|
|
WORK_FACTS_QUERY = public_query(WORK_FACTS_QUERY, nodes=("person",), documents=("document",))
|
|
THESIS_PATHS_QUERY = public_query(THESIS_PATHS_QUERY, nodes=("person",), documents=("document",))
|
|
WORK_PATHS_QUERY = public_query(WORK_PATHS_QUERY, nodes=("person",), documents=("document",))
|
|
START_YEAR_FACTS_QUERY = public_query(START_YEAR_FACTS_QUERY, nodes=("person",))
|
|
PERSON_KEYWORD_FACTS_QUERY = public_query(
|
|
PERSON_KEYWORD_FACTS_QUERY, nodes=("person",), documents=("document",)
|
|
)
|
|
PERSON_CATEGORY_FACTS_QUERY = public_query(
|
|
PERSON_CATEGORY_FACTS_QUERY, nodes=("person",), documents=("document",)
|
|
)
|
|
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(
|
|
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY, nodes=("person",), documents=("document",)
|
|
)
|
|
SHARED_KEYWORD_FACTS_QUERY = public_query(
|
|
SHARED_KEYWORD_FACTS_QUERY,
|
|
nodes=("left", "right"),
|
|
documents=("left_document", "right_document"),
|
|
)
|
|
SHARED_CATEGORY_FACTS_QUERY = public_query(
|
|
SHARED_CATEGORY_FACTS_QUERY,
|
|
nodes=("left", "right"),
|
|
documents=("left_document", "right_document"),
|
|
)
|
|
SHARED_DOCUMENT_AUTHOR_FACTS_QUERY = public_query(
|
|
SHARED_DOCUMENT_AUTHOR_FACTS_QUERY,
|
|
nodes=("left", "right"),
|
|
documents=("left_document", "right_document"),
|
|
)
|
|
DOCUMENT_SOURCES_QUERY = public_query(DOCUMENT_SOURCES_QUERY, documents=("document",))
|
|
|
|
|
|
def search_graph_with_session(
|
|
session: Any,
|
|
query: str,
|
|
*,
|
|
limit: int = DEFAULT_LIMIT,
|
|
max_entities: int = DEFAULT_MAX_ENTITIES,
|
|
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
|
question_plan=None,
|
|
) -> dict[str, Any]:
|
|
clean_query = str(query or "").strip()
|
|
if not clean_query:
|
|
raise ValueError("Query nesmie byť prázdny")
|
|
if limit < 1:
|
|
raise ValueError("Limit musí byť aspoň 1")
|
|
if max_entities < 1:
|
|
raise ValueError("max_entities musí byť aspoň 1")
|
|
|
|
constraints = extract_query_constraints(clean_query)
|
|
unsupported = unsupported_constraint(clean_query)
|
|
spec=None
|
|
if question_plan is not None:
|
|
from tools.answers.question_plan import QuestionPlan
|
|
spec=question_plan if isinstance(question_plan,QuestionPlan) else QuestionPlan.model_validate(question_plan)
|
|
# The shared planner implements work-year intervals. The legacy raw
|
|
# search guard must not reject that validated operator before execution.
|
|
# Record/event intervals remain unsupported by this metadata executor.
|
|
if unsupported=='year_range' and spec.period and spec.period.field=='work_year':
|
|
unsupported=None
|
|
if unsupported:
|
|
return {
|
|
**result_fields(
|
|
status="ambiguous", sources=[], reason="unsupported_constraint", complete=False
|
|
),
|
|
"query": clean_query,
|
|
"constraints": constraints,
|
|
"clarification": constraint_message(unsupported),
|
|
"facts": [],
|
|
"paths": [],
|
|
"documents": [],
|
|
"aggregation": None,
|
|
"resolved_entities": [],
|
|
"checked_entities": [],
|
|
"rejection_evidence": [],
|
|
"strategies": [],
|
|
"reasons": ["unsupported_constraint"],
|
|
}
|
|
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only)
|
|
original_query = clean_query
|
|
resolution = resolve_person_names(clean_query, catalog)
|
|
if resolution["status"] != "ok":
|
|
return {
|
|
**result_fields(
|
|
status=resolution["status"], sources=[], reason=resolution["reason"], complete=False
|
|
),
|
|
"query": original_query,
|
|
"constraints": constraints,
|
|
"name_resolution": resolution,
|
|
"clarification": resolution["message"] if resolution["status"] == "ambiguous" else None,
|
|
"facts": [],
|
|
"paths": [],
|
|
"documents": [],
|
|
"aggregation": None,
|
|
"resolved_entities": [],
|
|
"checked_entities": [],
|
|
"rejection_evidence": [],
|
|
"strategies": [],
|
|
"reasons": [resolution["reason"]],
|
|
}
|
|
clean_query = resolution["resolved_query"]
|
|
constraints = extract_query_constraints(clean_query)
|
|
if question_plan is not None:
|
|
if normalize_for_match(spec.question)!=normalize_for_match(clean_query):
|
|
raise ValueError('Question plan does not match the graph request')
|
|
if spec.intent!='metadata' or spec.unsupported:
|
|
raise ValueError('Only validated metadata plans belong to this executor')
|
|
# Graph templates and projections stay fixed; semantic work filters
|
|
# come from the same plan as the text and content-graph adapters.
|
|
constraints['years']=sorted({year for work in spec.works for year in work.years})
|
|
if spec.period and spec.period.field=='work_year':
|
|
first,last=int(spec.period.start[:4]),int(spec.period.end[:4])
|
|
constraints['years']=[y for y in (constraints['years'] or range(first,last+1)) if first<=y<=last]
|
|
kinds={work.work_type for work in spec.works if work.work_type}
|
|
constraints['thesis_types']=sorted(k for k in kinds if k.endswith('_thesis'))
|
|
constraints['work_types']=sorted(k for k in kinds if not k.endswith('_thesis'))
|
|
constraints['mentions_thesis']=bool(constraints['thesis_types'])
|
|
constraints['mentions_work']=bool(constraints['work_types'])
|
|
constraints['mentions_count']=spec.presentation=='count'
|
|
constraints['mentions_work_listing']=bool(
|
|
spec.presentation=='list' and spec.scope=='across_works'
|
|
and not set(spec.fields)&{'keyword','category','document_author','start_year'}
|
|
or 'work_type' in spec.fields and spec.scope=='across_works')
|
|
if spec.presentation=='count':
|
|
constraints['mentions_academic_year']=True # fetch academic records, not meetings
|
|
constraints["published_only"] = published_only
|
|
# Explicit tag filters are resolved separately. Incidental names such as the
|
|
# tag 'zp' in 'ZP Wiki' must never become additional query constraints.
|
|
phrase = constraints["keyword_phrase"]
|
|
entities = resolve_entities(
|
|
clean_query,
|
|
catalog,
|
|
max_entities=max(len(catalog), max_entities),
|
|
preferred_kinds=constraints["target_kinds"],
|
|
)
|
|
identity_paths = {
|
|
normalize_for_match(m["name"]): m["source_document"]
|
|
for m in resolution["matches"]
|
|
if m.get("source_document")
|
|
}
|
|
entities = [
|
|
row
|
|
for row in entities
|
|
if row.get("kind") != "Person"
|
|
or normalize_for_match(row.get("name", "")) not in identity_paths
|
|
or row.get("source_document") == identity_paths[normalize_for_match(row["name"])]
|
|
]
|
|
if phrase:
|
|
keywords = explicit_keyword_entities(phrase, catalog)
|
|
entities = [row for row in entities if row.get("kind") != "Keyword"] + keywords
|
|
constraints["unresolved_keyword"] = not bool(keywords)
|
|
else:
|
|
constraints["unresolved_keyword"] = False
|
|
if constraints["mentions_keyword"]:
|
|
entities = [row for row in entities if row.get("kind") != "Keyword"]
|
|
elif "zp wiki" in normalize_for_match(clean_query):
|
|
entities = [
|
|
row
|
|
for row in entities
|
|
if not (row.get("kind") == "Keyword" and row.get("matched_alias") == "zp")
|
|
]
|
|
|
|
author_mentions = (
|
|
named_author_phrases(clean_query) if constraints["mentions_document_author"] else []
|
|
)
|
|
requested_authors = [
|
|
{**row, "matched_alias": normalize_for_match(row.get("name", "")), "match_score": score}
|
|
for row in catalog
|
|
if row.get("kind") == "DocumentAuthor"
|
|
for requested in author_mentions
|
|
for score, _ in [person_name_match(requested, row.get("name", ""))]
|
|
if score > 0
|
|
]
|
|
if author_mentions:
|
|
author_names = {row["matched_alias"] for row in requested_authors}
|
|
entities = [
|
|
row
|
|
for row in entities
|
|
if row.get("kind") != "DocumentAuthor"
|
|
and not (row.get("kind") == "Person" and row.get("matched_alias") in author_names)
|
|
] + requested_authors
|
|
|
|
# The shared resolver has already rejected unknown/ambiguous subjects.
|
|
# Re-parsing prose here with a second role regex used to invent additional
|
|
# names from presentation instructions, despite successful resolution.
|
|
constraints["requested_person_names"] = list(
|
|
dict.fromkeys(normalize_for_match(match["name"]) for match in resolution["matches"])
|
|
)
|
|
constraints["unresolved_person_names"] = []
|
|
constraints["unresolved_document_author"] = bool(author_mentions and not requested_authors)
|
|
aliases_to_ids: dict[tuple[str, str], set[str]] = {}
|
|
for row in entities:
|
|
if row.get("kind") in {"Person", "DocumentAuthor"}:
|
|
key = (row["kind"], row["matched_alias"])
|
|
aliases_to_ids.setdefault(key, set()).add(str(row["element_id"]))
|
|
ambiguous_names = [name for (_, name), matches in aliases_to_ids.items() if len(matches) > 1]
|
|
constraints["ambiguous_person_names"] = sorted(set(ambiguous_names))
|
|
|
|
# Preserve explicitly mentioned people when a longer topic/title also matches.
|
|
entities.sort(key=lambda row: (row.get("kind") != "Person", -row["match_score"]))
|
|
resolved_entities = entities[:max_entities]
|
|
entity_limit_reached = len(entities) > max_entities
|
|
|
|
def ids(kind: str) -> list[str]:
|
|
return [
|
|
str(row["element_id"])
|
|
for row in resolved_entities
|
|
if row.get("kind") == kind and row.get("element_id")
|
|
]
|
|
|
|
person_ids = ids("Person")
|
|
thesis_ids, work_ids = ids("Thesis"), ids("Work")
|
|
keyword_ids, category_ids = ids("Keyword"), ids("Category")
|
|
author_ids = ids("DocumentAuthor")
|
|
year_specific = bool(constraints["years"] or constraints["mentions_academic_year"])
|
|
title_specific = constraints["mentions_title"]
|
|
academic_specific = title_specific or year_specific or constraints["mentions_work_listing"]
|
|
thesis_specific = (
|
|
bool(
|
|
constraints["mentions_thesis"]
|
|
or thesis_ids
|
|
or (academic_specific and not constraints["mentions_work"])
|
|
)
|
|
and not constraints["mentions_start_year"]
|
|
)
|
|
work_specific = (
|
|
bool(
|
|
constraints["mentions_work"]
|
|
or work_ids
|
|
or (academic_specific and not constraints["mentions_thesis"])
|
|
)
|
|
and not constraints["mentions_start_year"]
|
|
)
|
|
shared_keyword = bool(
|
|
constraints["mentions_shared"]
|
|
and len(person_ids) >= 2
|
|
and (constraints["mentions_keyword"] or constraints["mentions_topic"])
|
|
)
|
|
shared_category = bool(
|
|
constraints["mentions_shared"] and len(person_ids) >= 2 and constraints["mentions_category"]
|
|
)
|
|
shared_author = bool(
|
|
constraints["mentions_shared"]
|
|
and len(person_ids) >= 2
|
|
and constraints["mentions_document_author"]
|
|
)
|
|
keyword_specific = bool(
|
|
not shared_keyword and (keyword_ids or (person_ids and constraints["mentions_keyword"]))
|
|
)
|
|
category_specific = bool(
|
|
not shared_category and (category_ids or (person_ids and constraints["mentions_category"]))
|
|
)
|
|
author_specific = bool(not shared_author and constraints["mentions_document_author"])
|
|
structured = any(
|
|
(
|
|
thesis_specific,
|
|
work_specific,
|
|
constraints["mentions_start_year"],
|
|
keyword_specific,
|
|
category_specific,
|
|
author_specific,
|
|
shared_keyword,
|
|
shared_category,
|
|
shared_author,
|
|
)
|
|
)
|
|
# Fetch complete structured evidence before filtering/counting and applying
|
|
# the presentation limit. The catalog bounds the number of academic nodes;
|
|
# relational lookups use a conservative bound covering the managed graph.
|
|
fetch_limit = max(limit, len(catalog) ** 2)
|
|
common = {
|
|
"scope": GRAPH_SCOPE,
|
|
"person_ids": person_ids,
|
|
"limit": fetch_limit,
|
|
"published_only": published_only,
|
|
}
|
|
blocked_reasons = []
|
|
# A written year outside the current corpus is still a real constraint.
|
|
# Malformed/range notation is explicitly unsupported, never a global query.
|
|
explicit_year_tokens = re.findall(r"\b(?:roku|rokoch)\s+([^\s,?!]+)", clean_query, flags=re.I)
|
|
if any(
|
|
re.match(r"\d", token) and not re.fullmatch(r"\d{4}[.]?", token)
|
|
for token in explicit_year_tokens
|
|
):
|
|
blocked_reasons.append("unsupported_year_constraint")
|
|
if constraints["mentions_start_year"] and any(
|
|
row.get("kind") == "Person" and row.get("start_year") is None for row in resolved_entities
|
|
):
|
|
blocked_reasons.append("missing_documented_start_year")
|
|
if constraints["unresolved_keyword"]:
|
|
blocked_reasons.append("unknown_keyword")
|
|
if constraints["unresolved_document_author"]:
|
|
blocked_reasons.append("unknown_document_author")
|
|
if ambiguous_names:
|
|
blocked_reasons.append("ambiguous_person")
|
|
if constraints["mentions_count"] and not (thesis_specific or work_specific):
|
|
blocked_reasons.append("unsupported_count_domain")
|
|
# A quoted unknown work title is a failed exact lookup, not a global listing.
|
|
quoted_titles = [
|
|
match[1]
|
|
for match in re.finditer(r'[„“"«]([^„“”"»]+)[“”"»]', clean_query)
|
|
if not re.search(
|
|
r"\b(?:tag|tagom|tagu|klucov\w*\s+slov\w*)$",
|
|
normalize_for_match(clean_query[: match.start()]),
|
|
)
|
|
]
|
|
from tools.retrieval.work_selector import HEADING,label_key
|
|
quoted_titles=[title for title in quoted_titles if not HEADING.fullmatch(label_key(title))]
|
|
if quoted_titles and (thesis_specific or work_specific) and not thesis_ids and not work_ids:
|
|
blocked_reasons.append("unknown_title")
|
|
constraints["unresolved_title"] = True
|
|
blocked = bool(blocked_reasons)
|
|
fetch_capped = False
|
|
|
|
def fetch(enabled: bool, cypher: str, **params: Any) -> list[dict[str, Any]]:
|
|
nonlocal fetch_capped
|
|
if not enabled or blocked:
|
|
return []
|
|
rows = run_rows(session, cypher, **{**common, "limit": fetch_limit + 1}, **params)
|
|
fetch_capped = fetch_capped or len(rows) > fetch_limit
|
|
return rows[:fetch_limit]
|
|
|
|
thesis_rows = fetch(
|
|
thesis_specific,
|
|
THESIS_FACTS_QUERY,
|
|
thesis_ids=thesis_ids,
|
|
years=constraints["years"],
|
|
thesis_types=constraints["thesis_types"],
|
|
)
|
|
work_rows = fetch(
|
|
work_specific,
|
|
WORK_FACTS_QUERY,
|
|
work_ids=work_ids,
|
|
years=constraints["years"],
|
|
work_types=constraints["work_types"],
|
|
)
|
|
if question_plan is not None:
|
|
thesis_rows=[row for row in thesis_rows if spec.matches_work(row.get('person'),row.get('heading'),row.get('thesis_type'),row.get('year'))]
|
|
work_rows=[row for row in work_rows if spec.matches_work(row.get('person'),row.get('heading'),row.get('work_type'),row.get('year'))]
|
|
from tools.answers.question_plan import ambiguous_work_choice
|
|
message=ambiguous_work_choice(spec,[{'person':r.get('person'),'work_label':r.get('heading')} for r in thesis_rows+work_rows])
|
|
if message:
|
|
return {**result_fields(status='ambiguous',sources=[],reason='ambiguous_work',complete=False),
|
|
'query':original_query,'constraints':constraints,'clarification':message,
|
|
'name_resolution':resolution,'facts':[],'paths':[],'documents':[],'aggregation':None,
|
|
'resolved_entities':entities,'strategies':[],'checked_entities':[],
|
|
'rejection_evidence':[],'reasons':['ambiguous_work']}
|
|
if title_specific and not constraints["mentions_work_listing"] and not person_ids:
|
|
thesis_rows = [row for row in thesis_rows if row.get("title")]
|
|
work_rows = [row for row in work_rows if row.get("title")]
|
|
start_rows = fetch(
|
|
constraints["mentions_start_year"], START_YEAR_FACTS_QUERY, years=constraints["years"]
|
|
)
|
|
keyword_rows = fetch(keyword_specific, PERSON_KEYWORD_FACTS_QUERY, keyword_ids=keyword_ids)
|
|
category_rows = fetch(category_specific, PERSON_CATEGORY_FACTS_QUERY, category_ids=category_ids)
|
|
author_rows = fetch(
|
|
author_specific,
|
|
PERSON_DOCUMENT_AUTHOR_FACTS_QUERY,
|
|
author_ids=author_ids,
|
|
category_ids=category_ids,
|
|
)
|
|
shared_keyword_rows = fetch(shared_keyword, SHARED_KEYWORD_FACTS_QUERY)
|
|
if keyword_ids:
|
|
shared_keyword_rows = [
|
|
row for row in shared_keyword_rows if str(row.get("keyword_element_id")) in keyword_ids
|
|
]
|
|
shared_category_rows = fetch(shared_category, SHARED_CATEGORY_FACTS_QUERY)
|
|
if category_ids:
|
|
shared_category_rows = [
|
|
row
|
|
for row in shared_category_rows
|
|
if str(row.get("category_element_id")) in category_ids
|
|
]
|
|
shared_author_rows = fetch(shared_author, SHARED_DOCUMENT_AUTHOR_FACTS_QUERY)
|
|
|
|
# Conjunctions must be witnessed by the same person, not by unrelated rows.
|
|
if keyword_specific and (thesis_specific or work_specific):
|
|
matching_people = {row.get("person_element_id") for row in keyword_rows}
|
|
thesis_rows = [
|
|
row for row in thesis_rows if row.get("person_element_id") in matching_people
|
|
]
|
|
work_rows = [row for row in work_rows if row.get("person_element_id") in matching_people]
|
|
academic_people = {row.get("person_element_id") for row in thesis_rows + work_rows}
|
|
keyword_rows = [
|
|
row for row in keyword_rows if row.get("person_element_id") in academic_people
|
|
]
|
|
|
|
shared_years: list[int] | None = None
|
|
if (
|
|
not blocked
|
|
and constraints["mentions_academic_year"]
|
|
and constraints["mentions_shared"]
|
|
and len(person_ids) >= 2
|
|
and not (shared_keyword or shared_category or shared_author)
|
|
):
|
|
years_by_person = {
|
|
person: {
|
|
row["year"]
|
|
for row in thesis_rows + work_rows
|
|
if row.get("person_element_id") == person and row.get("year") is not None
|
|
}
|
|
for person in person_ids
|
|
}
|
|
shared_years = sorted(set.intersection(*years_by_person.values()))
|
|
thesis_rows = [row for row in thesis_rows if row.get("year") in shared_years]
|
|
work_rows = [row for row in work_rows if row.get("year") in shared_years]
|
|
|
|
paths: list[dict[str, Any]] = []
|
|
path_capped = False
|
|
for enabled, cypher, rows, kind, node_ids, types in (
|
|
(
|
|
thesis_specific,
|
|
THESIS_PATHS_QUERY,
|
|
thesis_rows,
|
|
"Thesis",
|
|
thesis_ids,
|
|
constraints["thesis_types"],
|
|
),
|
|
(work_specific, WORK_PATHS_QUERY, work_rows, "Work", work_ids, constraints["work_types"]),
|
|
):
|
|
params = (
|
|
{"thesis_ids": node_ids, "thesis_types": types}
|
|
if kind == "Thesis"
|
|
else {"work_ids": node_ids, "work_types": types}
|
|
)
|
|
returned = fetch(enabled, cypher, years=constraints["years"], **params)
|
|
allowed = {str(row.get("thesis_element_id") or row.get("work_element_id")) for row in rows}
|
|
paths.extend(
|
|
path
|
|
for path in returned
|
|
if any(
|
|
node.get("kind") == kind and str(node.get("element_id")) in allowed
|
|
for node in path.get("nodes", [])
|
|
)
|
|
)
|
|
if not structured and resolved_entities and not blocked:
|
|
start_ids = [str(row["element_id"]) for row in resolved_entities]
|
|
params = {
|
|
"scope": GRAPH_SCOPE,
|
|
"start_ids": start_ids,
|
|
"limit": limit + 1,
|
|
"published_only": published_only,
|
|
}
|
|
if constraints["mentions_shared"] and len(start_ids) >= 2:
|
|
paths.extend(run_rows(session, PAIR_PATH_QUERY, **params))
|
|
else:
|
|
paths.extend(
|
|
run_rows(
|
|
session, NEIGHBORHOOD_QUERY, target_kinds=constraints["target_kinds"], **params
|
|
)
|
|
)
|
|
path_capped = len(paths) > limit
|
|
paths = deduplicate_paths(paths)
|
|
facts = build_facts(
|
|
thesis_rows,
|
|
work_rows,
|
|
start_rows,
|
|
keyword_rows,
|
|
category_rows,
|
|
shared_keyword_rows,
|
|
shared_category_rows,
|
|
shared_author_rows,
|
|
mentions_shared=constraints["mentions_shared"],
|
|
)
|
|
for row in author_rows:
|
|
facts.append({**row, "fact_type": "person_document_author"})
|
|
if row.get("category") and not any(
|
|
fact.get("fact_type") == "person_category"
|
|
and fact.get("person_element_id") == row.get("person_element_id")
|
|
and fact.get("document_path") == row.get("document_path")
|
|
and fact.get("category") == row.get("category")
|
|
for fact in facts
|
|
):
|
|
facts.append({**row, "fact_type": "person_category"})
|
|
if shared_years is not None:
|
|
facts.append(
|
|
{
|
|
"fact_type": "academic_year_comparison",
|
|
"shared_years": shared_years,
|
|
"same_academic_year": bool(shared_years),
|
|
"people": [row["name"] for row in resolved_entities if row.get("kind") == "Person"],
|
|
}
|
|
)
|
|
|
|
aggregation = None
|
|
if constraints["mentions_count"] and not blocked:
|
|
matching = sorted({str(row["person"]) for row in thesis_rows + work_rows})
|
|
record_count=bool(re.search(r'\b(?:kolko|pocet)\s+(?:roznych\s+|evidovanych\s+)?(?:prac\w*|projekt\w*)\b',normalize_for_match(clean_query)))
|
|
records={row.get('thesis_id') or row.get('work_id') or row.get('thesis_element_id') or row.get('work_element_id') for row in thesis_rows+work_rows}
|
|
records.discard(None)
|
|
aggregation = {
|
|
"count": len(records) if record_count else len(matching),
|
|
"count_domain":'academic_records' if record_count else 'people',
|
|
"people": matching,
|
|
"candidate_count": len(person_ids),
|
|
"complete": not (entity_limit_reached or fetch_capped),
|
|
}
|
|
# Sources of rejected candidates still document the comparison. Fetch their
|
|
# actual stored URLs rather than manufacturing URLs from expected answers.
|
|
documents = collect_documents(resolved_entities, paths, facts)
|
|
sources = (
|
|
run_rows(
|
|
session,
|
|
DOCUMENT_SOURCES_QUERY,
|
|
scope=GRAPH_SCOPE,
|
|
paths=[row["document_path"] for row in documents],
|
|
published_only=published_only,
|
|
)
|
|
if documents
|
|
else []
|
|
)
|
|
by_path = {row["document_path"]: row.get("source_url") for row in sources}
|
|
if published_only:
|
|
documents = [row for row in documents if row["document_path"] in by_path]
|
|
for document in documents:
|
|
if not document.get("source_url"):
|
|
document["source_url"] = by_path.get(document["document_path"])
|
|
|
|
strategies = []
|
|
for active, name in (
|
|
(resolved_entities, "entity_resolution"),
|
|
(paths, "graph_traversal"),
|
|
(thesis_rows, "thesis_constraints"),
|
|
(work_rows, "work_constraints"),
|
|
(start_rows, "start_year_constraints"),
|
|
(keyword_rows, "person_keyword_constraints"),
|
|
(category_rows, "person_category_constraints"),
|
|
(author_rows, "person_document_author_constraints"),
|
|
(shared_keyword_rows, "shared_keyword_constraints"),
|
|
(shared_category_rows, "shared_category_constraints"),
|
|
(shared_author_rows, "shared_document_author_constraints"),
|
|
):
|
|
if active:
|
|
strategies.append(name)
|
|
positive_facts = [
|
|
row
|
|
for row in facts
|
|
if row.get("fact_type") not in {"academic_year_comparison", "start_year_comparison"}
|
|
]
|
|
status = (
|
|
"ambiguous"
|
|
if ambiguous_names
|
|
else (
|
|
"no_answer"
|
|
if blocked or not (positive_facts or paths or aggregation is not None)
|
|
else "ok"
|
|
)
|
|
)
|
|
reason = (
|
|
blocked_reasons[0]
|
|
if blocked_reasons
|
|
else ("no_matching_facts" if status == "no_answer" else None)
|
|
)
|
|
truncation_reasons = []
|
|
for active, name in (
|
|
(entity_limit_reached, "entity_limit"),
|
|
(fetch_capped, "evidence_limit"),
|
|
(len(facts) > limit, "fact_limit"),
|
|
(path_capped or len(paths) > limit, "path_limit"),
|
|
):
|
|
if active:
|
|
truncation_reasons.append(name)
|
|
titles = {row["document_path"]: row.get("title") for row in sources}
|
|
citations = source_references(
|
|
[{**doc, "title": titles.get(doc["document_path"])} for doc in documents]
|
|
)
|
|
visible_facts = facts[:limit]
|
|
# A display cap must not separate an academic row from the tag proof that
|
|
# licensed its selection. These are supporting predicates for visible
|
|
# records, never additional people/academic records in the visible list.
|
|
visible_academic_people = {
|
|
row.get("person_element_id")
|
|
for row in visible_facts
|
|
if row.get("fact_type") in {"person_thesis", "person_work"} and row.get("person_element_id")
|
|
}
|
|
supporting_facts = [
|
|
row
|
|
for row in facts[limit:]
|
|
if keyword_specific
|
|
and (thesis_specific or work_specific)
|
|
and row.get("fact_type") == "person_keyword"
|
|
and row.get("person_element_id") in visible_academic_people
|
|
]
|
|
if len(visible_facts) + len(supporting_facts) == len(facts):
|
|
# Moving predicate evidence to its support field loses no result.
|
|
truncation_reasons = [reason for reason in truncation_reasons if reason != "fact_limit"]
|
|
return {
|
|
**result_fields(
|
|
status=status,
|
|
sources=citations,
|
|
reason=reason,
|
|
complete=True,
|
|
truncation_reasons=truncation_reasons,
|
|
),
|
|
"reasons": blocked_reasons,
|
|
"query": original_query,
|
|
"resolved_query": clean_query,
|
|
"name_resolution": resolution,
|
|
"strategies": strategies,
|
|
"constraints": constraints,
|
|
"resolved_entities": resolved_entities,
|
|
"paths": paths[:limit],
|
|
"facts": visible_facts,
|
|
"supporting_facts": supporting_facts,
|
|
"documents": documents,
|
|
"aggregation": aggregation,
|
|
"academic_year_comparison": (
|
|
{
|
|
"shared_years": shared_years,
|
|
"people": [row["name"] for row in resolved_entities if row.get("kind") == "Person"],
|
|
"complete": not (entity_limit_reached or fetch_capped),
|
|
}
|
|
if shared_years is not None
|
|
else None
|
|
),
|
|
"checked_entities": resolved_entities,
|
|
"rejection_evidence": [
|
|
row
|
|
for row in facts
|
|
if row.get("fact_type") in {"academic_year_comparison", "start_year_comparison"}
|
|
and not positive_facts
|
|
],
|
|
}
|
|
|
|
|
|
def search_graph(
|
|
query: str,
|
|
*,
|
|
limit: int = (DEFAULT_LIMIT),
|
|
max_entities: int = (DEFAULT_MAX_ENTITIES),
|
|
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
|
question_plan=None,
|
|
) -> dict[str, Any]:
|
|
from .graph_db import (
|
|
create_neo4j_driver,
|
|
get_neo4j_settings,
|
|
)
|
|
|
|
from tools.sync.index_state import DATA_DIR, IndexUnavailableError, index_snapshot
|
|
from .build_graphrag import graph_version
|
|
|
|
settings = get_neo4j_settings()
|
|
|
|
with index_snapshot(DATA_DIR) as version:
|
|
if published_only and (
|
|
version is None or version.get("publication_policy") != PUBLICATION_POLICY_VERSION
|
|
):
|
|
raise IndexUnavailableError("Graf vyžaduje synchronizáciu pravidiel publikovania.")
|
|
with create_neo4j_driver(settings) as driver:
|
|
driver.verify_connectivity()
|
|
with driver.session(database=settings.database) as session:
|
|
if version is not None and graph_version(session) != version:
|
|
raise IndexUnavailableError("Neo4j graf nezodpovedá aktívnej verzii dát.")
|
|
result = search_graph_with_session(
|
|
session,
|
|
query,
|
|
limit=limit,
|
|
max_entities=max_entities,
|
|
published_only=published_only,
|
|
question_plan=question_plan,
|
|
)
|
|
if version is not None:
|
|
result["index_version"] = version
|
|
return result
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument("query")
|
|
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=DEFAULT_LIMIT,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-entities",
|
|
type=int,
|
|
default=(DEFAULT_MAX_ENTITIES),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--include-unpublished",
|
|
action="store_true",
|
|
help="Výslovne zahrnie aj nepublikované dokumenty.",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
result = search_graph(
|
|
args.query,
|
|
limit=args.limit,
|
|
max_entities=args.max_entities,
|
|
published_only=not args.include_unpublished,
|
|
)
|
|
except Exception as error:
|
|
result = error_result(error)
|
|
|
|
print(
|
|
json.dumps(
|
|
result,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
if result["status"] == "error":
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|