481 lines
18 KiB
Python
481 lines
18 KiB
Python
"""Conservative, catalog-bound person resolution shared by both retrieval engines.
|
|
|
|
Only contiguous full names may be corrected automatically. Fuzzy matches must
|
|
have an exact/inflected anchor, at most one misspelled token and a unique best
|
|
catalog identity. An unresolved explicit subject never means an empty filter.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import defaultdict
|
|
import re
|
|
import sqlite3
|
|
import unicodedata
|
|
from contextlib import closing
|
|
|
|
|
|
def normalize(value: str) -> str:
|
|
text = unicodedata.normalize("NFKD", value).casefold()
|
|
return " ".join(
|
|
re.findall(r"[^\W_]+", "".join(c for c in text if not unicodedata.combining(c)))
|
|
)
|
|
|
|
|
|
def name_token_variants(token: str) -> set[str]:
|
|
variants = {token}
|
|
if len(token) < 3:
|
|
return variants
|
|
irregular = {
|
|
"peter": ("petra", "petrovi", "petrom"),
|
|
"pavol": ("pavla", "pavlovi", "pavlom"),
|
|
"frantisek": ("frantiska", "frantiskovi", "frantiskom"),
|
|
}
|
|
variants.update(irregular.get(token, ()))
|
|
if token.endswith(('ii', 'yi')):
|
|
variants.add(token + 'ho')
|
|
if token.endswith("ova"):
|
|
variants.update(token[:-1] + s for s in ("ej", "u", "ou"))
|
|
elif token.endswith("y"):
|
|
variants.update(token[:-1] + s for s in ("eho", "emu", "ym", "ych"))
|
|
elif token.endswith("a"):
|
|
variants.update(token[:-1] + s for s in ("y", "e", "u", "ou", "ovi", "om"))
|
|
elif token.endswith("o"):
|
|
variants.update(token[:-1] + s for s in ("a", "ovi", "om"))
|
|
else:
|
|
variants.update(token + s for s in ("a", "ovi", "om"))
|
|
return variants
|
|
|
|
|
|
def _distance(a: str, b: str) -> int:
|
|
"""Optimal string alignment distance, including adjacent transpositions."""
|
|
rows = [list(range(len(b) + 1))]
|
|
for i, left in enumerate(a, 1):
|
|
row = [i]
|
|
for j, right in enumerate(b, 1):
|
|
value = min(row[-1] + 1, rows[-1][j] + 1, rows[-1][j - 1] + (left != right))
|
|
if i > 1 and j > 1 and left == b[j - 2] and a[i - 2] == right:
|
|
value = min(value, rows[-2][j - 2] + 1)
|
|
row.append(value)
|
|
rows.append(row)
|
|
return rows[-1][-1]
|
|
|
|
|
|
def _score(words: list[str], canonical: list[str]) -> int | None:
|
|
if len(words) != len(canonical):
|
|
return None
|
|
scores = []
|
|
for actual, expected in zip(words, canonical):
|
|
if actual == expected:
|
|
scores.append(0)
|
|
elif actual in name_token_variants(expected):
|
|
scores.append(1)
|
|
else:
|
|
if (
|
|
not actual.isalpha()
|
|
or not expected.isalpha()
|
|
or min(len(actual), len(expected)) < 3
|
|
):
|
|
return None
|
|
cap = 2 if min(len(actual), len(expected)) >= 8 else 1
|
|
forms = name_token_variants(expected)
|
|
distance = min(
|
|
(_distance(actual, form) for form in forms if abs(len(actual) - len(form)) <= cap),
|
|
default=cap + 1,
|
|
)
|
|
if distance > cap:
|
|
return None
|
|
scores.append(10 + distance)
|
|
# Never assemble a person from two approximate tokens.
|
|
if sum(s >= 10 for s in scores) > 1 or all(s >= 10 for s in scores):
|
|
return None
|
|
return sum(scores)
|
|
|
|
|
|
_STOP = set(
|
|
"a aj ale alebo ak ako ake aky aku ktore ktori ktora ktory kto kolko ci co obaja obidvaja "
|
|
"ma maju mal mala malo mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do rok roku rokoch "
|
|
"praca prace pracu projekt projekty dokument dokumenty nazov nazvy nazv "
|
|
"tag tagy klucove slova slovo temu temy kategoriu kategorie "
|
|
"uved uvedte vypis vyber najdi zisti porovnaj popis opis čomu komu vymenuj pomenuj "
|
|
"vsetci vsetky vsetkych plan stav vysledky problemy metody experimenty "
|
|
"osoba osoby osobu osobe osobach student studenti studentov autora autor "
|
|
"aktualne teraz vtedy predtym potom vedie viedol pokrocil pokrocili kedy prislusne o com pod "
|
|
"urobit urobil dalej dalsi dalsie dalsiu dalsich".split()
|
|
)
|
|
_STOP = {normalize(word) for word in _STOP}
|
|
|
|
|
|
def _generic(word: str) -> bool:
|
|
return word in _STOP or word.startswith(
|
|
(
|
|
"diplomov",
|
|
"bakalarsk",
|
|
"dizertacn",
|
|
"timov",
|
|
"zaverecn",
|
|
"dokument",
|
|
"praci",
|
|
"suvisi",
|
|
"patriac",
|
|
"osob",
|
|
"student",
|
|
"kandidat",
|
|
"kategori",
|
|
"zameran",
|
|
"ciel",
|
|
"vysled",
|
|
"priebeh",
|
|
"pouz",
|
|
"experiment",
|
|
"problem",
|
|
"metod",
|
|
"riesen",
|
|
"system",
|
|
"vyhladav",
|
|
"inform",
|
|
"studij",
|
|
"databaz",
|
|
"zdroj",
|
|
"prehlad",
|
|
"presn",
|
|
"postup",
|
|
"hodnot",
|
|
"technol",
|
|
"zadani",
|
|
"uloh",
|
|
"spaja",
|
|
"evidov",
|
|
"veden",
|
|
"veduc",
|
|
"konzult",
|
|
"skolitel",
|
|
"ukonc",
|
|
"skonc",
|
|
"uspes",
|
|
"neuspes",
|
|
"obhaj",
|
|
"studium",
|
|
"studia",
|
|
"stretnut",
|
|
"posledn",
|
|
)
|
|
)
|
|
|
|
|
|
def _identity(row: dict) -> tuple:
|
|
return (
|
|
normalize(row["name"]),
|
|
row.get("kind", "Person"),
|
|
row.get("source_document") or row.get("element_id") or row.get("id") or row["name"],
|
|
)
|
|
|
|
|
|
def resolve_person_names(query: str, catalog: list[dict]) -> dict:
|
|
"""Return an internal canonical query, public matches and fail-closed choices.
|
|
|
|
query itself stays unchanged for HTTP/MCP echo binding. Replacements affect
|
|
name character spans only: work types, years, tags and negations survive.
|
|
"""
|
|
people = list(
|
|
{
|
|
_identity(row): row
|
|
for row in catalog
|
|
if row.get("name")
|
|
and len(normalize(row["name"]).split()) >= 2
|
|
and row.get("kind", "Person") in {"Person", "DocumentAuthor"}
|
|
}.values()
|
|
)
|
|
profile_names = {
|
|
normalize(row["name"]) for row in people if row.get("kind", "Person") == "Person"
|
|
}
|
|
author_lookup = bool(
|
|
re.search(
|
|
r"\b(?:autor\w*|dokument\w*|vedie|viedol|veduc\w*|veden\w*|skolitel\w*|konzult\w*)\b", normalize(query)
|
|
)
|
|
)
|
|
people = [
|
|
row
|
|
for row in people
|
|
if row.get("kind") != "DocumentAuthor"
|
|
or author_lookup
|
|
and normalize(row["name"]) not in profile_names
|
|
]
|
|
tokens = list(re.finditer(r"[^\W_]+", query, re.UNICODE))
|
|
words = [normalize(token[0]) for token in tokens]
|
|
quoted = [(m.start(), m.end()) for m in re.finditer(r'["„“«][^"„“”»]+["“”»]', query)]
|
|
opaque = [
|
|
(m.start(), m.end())
|
|
for m in re.finditer(r"(?:https?://|(?:pages/)?(?:students|interns)/)[^\s]+", query)
|
|
]
|
|
# A tag/title can itself look like a person's name. Those literals are
|
|
# constraints, not an invitation to correct a student identity.
|
|
literal = [
|
|
(m.start(1), m.end(1))
|
|
for m in re.finditer(
|
|
r"\b(?:tag(?:om|u)?|kľúčov\w*\s+slov\w*|klucov\w*\s+slov\w*|názvom|nazvom)\s+"
|
|
r"([^,;?!\n]+?)(?=\s+(?:a|pri|v\s+roku|z\s+roku|ktor\w*)\b|[,;?!\n]|$)",
|
|
query,
|
|
re.I,
|
|
)
|
|
if normalize(m[1]).split()[0]
|
|
not in {"ma", "maju", "mal", "mali", "je", "su", "bol", "boli", "uved", "vypis"}
|
|
]
|
|
|
|
def inside_quote(start, end):
|
|
return any(left <= start and end <= right for left, right in [*quoted, *opaque, *literal])
|
|
|
|
spans = defaultdict(list)
|
|
for row in people:
|
|
canonical = normalize(row["name"]).split()
|
|
orders = [canonical]
|
|
if len(canonical) >= 2:
|
|
orders.append([canonical[-1], *canonical[:-1]])
|
|
for i in range(len(tokens) - len(canonical) + 1):
|
|
start, end = tokens[i].start(), tokens[i + len(canonical) - 1].end()
|
|
if inside_quote(start, end):
|
|
continue
|
|
gaps = [
|
|
query[tokens[j].end() : tokens[j + 1].start()]
|
|
for j in range(i, i + len(canonical) - 1)
|
|
]
|
|
if any(not re.fullmatch(r"[\s,]+", gap) for gap in gaps):
|
|
continue
|
|
scores = [
|
|
(score, order != canonical)
|
|
for order in orders
|
|
if (score := _score(words[i : i + len(canonical)], order)) is not None
|
|
]
|
|
if scores:
|
|
score, reversed_order = min(scores)
|
|
spans[(start, end)].append((score, row, reversed_order))
|
|
|
|
selected = []
|
|
for (start, end), options in sorted(
|
|
spans.items(),
|
|
key=lambda item: (min(v[0] for v in item[1]), -(item[0][1] - item[0][0]), item[0][0]),
|
|
):
|
|
if any(start < right and left < end for left, right, _ in selected):
|
|
continue
|
|
best = min(value[0] for value in options)
|
|
# A complete literal name wins. Otherwise every bounded plausible
|
|
# identity is a choice: the number of inflected tokens is not evidence
|
|
# that one real person is more likely than another.
|
|
choices = [v for v in options if v[0] == 0] if best == 0 else options
|
|
explicit_profiles = [
|
|
v for v in choices if v[1].get("source_document") and v[1]["source_document"] in query
|
|
]
|
|
if len(explicit_profiles) == 1:
|
|
choices = explicit_profiles
|
|
# A document author and student may share a name without being two
|
|
# student identities. Keep roles separate in graph constraints later.
|
|
by_name = defaultdict(list)
|
|
for value in choices:
|
|
by_name[normalize(value[1]["name"])].append(value)
|
|
duplicate = any(
|
|
len({_identity(v[1]) for v in group if v[1].get("kind", "Person") == "Person"}) > 1
|
|
for group in by_name.values()
|
|
)
|
|
if len(by_name) > 1 or duplicate:
|
|
return _blocked(
|
|
query, "ambiguous", "ambiguous_person", query[start:end], [v[1] for v in choices]
|
|
)
|
|
choices.sort(key=lambda v: v[1].get("kind", "Person") != "Person")
|
|
selected.append((start, end, choices[0]))
|
|
|
|
def covered(start, end):
|
|
return any(left <= start and end <= right for left, right, _ in selected)
|
|
|
|
first_names = {
|
|
variant
|
|
for row in people
|
|
for variant in name_token_variants(normalize(row["name"]).split()[0])
|
|
}
|
|
title_starts = {
|
|
variant
|
|
for row in catalog
|
|
if row.get("kind") not in {"Person", "DocumentAuthor", None}
|
|
for key in ("name", "title", "heading")
|
|
if row.get(key) and normalize(row[key])
|
|
for variant in name_token_variants(normalize(row[key]).split()[0])
|
|
}
|
|
first_subject = min((left for left, _, _ in selected), default=len(query))
|
|
list_prefix = normalize(re.split(r"[.!?;\n]", query[:first_subject])[-1])
|
|
explicit_list = bool(
|
|
re.search(r"\b(?:kandidat(?:ov|i)|osob(?:y|ach)?|student(?:i|ov))\s*$", list_prefix)
|
|
)
|
|
# Explicit subjects: full capitalized names; known given names followed by
|
|
# an unknown surname; names after a role, possessive work noun or predicate.
|
|
for i, token in enumerate(tokens):
|
|
if (
|
|
not any(c.isalpha() for c in words[i])
|
|
or _generic(words[i])
|
|
or inside_quote(token.start(), token.end())
|
|
or covered(token.start(), token.end())
|
|
):
|
|
continue
|
|
# A predicate in an earlier sentence cannot introduce this token:
|
|
# "uveď práce. Pridaj zdroje" must not create a person named Pridaj.
|
|
prefix = normalize(re.split(r"[.!?;\n]", query[: token.start()])[-1])
|
|
role = bool(
|
|
re.search(
|
|
r"\b(?:osob\w*|student\w*|kandidat\w*|autor(?:a|om)?|ma|mal|robil|riesil|venuje|prace|pracu|projektu)\s*$",
|
|
prefix,
|
|
)
|
|
)
|
|
# "Pri každej osobe príslušné práce" refers back to the subjects;
|
|
# it does not introduce a person named "príslušné". Do not use this
|
|
# quantified role as evidence of a new name. Independent full-name
|
|
# and known-given-name checks below still detect additional subjects.
|
|
reference = bool(
|
|
re.search(
|
|
r"\b(?:kazd\w*|jednotliv\w*|vsetk\w*|uveden\w*|vybran\w*|dan\w*|"
|
|
r"obe|oboch|obom|oboma|obidv\w*|tieto|tychto|tymto)\s+"
|
|
r"(?:osob\w*|student\w*|kandidat\w*|autor\w*)\s*$",
|
|
prefix,
|
|
)
|
|
)
|
|
role = role and not reference
|
|
following = i + 1 < len(tokens) and re.fullmatch(
|
|
r"\s+", query[token.end() : tokens[i + 1].start()]
|
|
)
|
|
pair = bool(following and not _generic(words[i + 1]))
|
|
upper_pair = bool(
|
|
pair
|
|
and token[0][0].isupper()
|
|
and tokens[i + 1][0][0].isupper()
|
|
and not token[0].isupper()
|
|
and not tokens[i + 1][0].isupper()
|
|
)
|
|
# A conjunction introduces another identity only while enumerating
|
|
# names. "Ján Holp ... a nezamieňaj ich" is a presentation request.
|
|
adjacent_name = any(right <= token.start() and re.fullmatch(
|
|
r"\s*(?:,|a)\s*", query[right:token.start()]) for _,right,_ in selected)
|
|
comparison_subject = bool(selected and pair and (adjacent_name or re.fullmatch(r"porovnaj\s*",prefix)))
|
|
identity_role = bool(re.search(r"\b(?:osob\w*|student\w*|kandidat\w*|autor(?:a|om)?)\s*$",prefix)) and not reference
|
|
owner_slot = role and not selected and not re.match(
|
|
r'^(?:kto|ktori|ktory|u koho)\b', normalize(query))
|
|
# Lower-case unknown entries in an explicit candidate list must still
|
|
# fail closed, even if they follow a comma or a known person.
|
|
list_subject = explicit_list and any(
|
|
right <= token.start() and re.fullmatch(r"\s*(?:,|a)\s*", query[right : token.start()])
|
|
for _, right, _ in selected
|
|
)
|
|
if not (
|
|
owner_slot or (identity_role and pair)
|
|
or words[i] in first_names
|
|
or upper_pair
|
|
or comparison_subject
|
|
or list_subject
|
|
):
|
|
continue
|
|
end = tokens[i + 1].end() if pair else token.end()
|
|
if covered(token.start(), end):
|
|
continue
|
|
mention = query[token.start() : end]
|
|
if not pair and words[i] in title_starts:
|
|
continue
|
|
if not pair:
|
|
# Partial names are never fuzzy-expanded. Offer actual catalog choices.
|
|
options = [
|
|
row
|
|
for row in people
|
|
if any(
|
|
words[i] in name_token_variants(part) for part in normalize(row["name"]).split()
|
|
)
|
|
]
|
|
if options:
|
|
return _blocked(query, "ambiguous", "incomplete_person_name", mention, options)
|
|
return _blocked(query, "no_answer", "unknown_person", mention, [])
|
|
|
|
resolved = query
|
|
matches = []
|
|
for start, end, (score, row, reverse) in sorted(selected, reverse=True):
|
|
resolved = resolved[:start] + row["name"] + resolved[end:]
|
|
matches.append(
|
|
{
|
|
"input": query[start:end],
|
|
"name": row["name"],
|
|
"method": "typo" if score >= 10 else "reordered" if reverse else "variant",
|
|
"source_document": row.get("source_document"),
|
|
}
|
|
)
|
|
matches.reverse()
|
|
notices = [
|
|
f'Rozpoznané meno: **{match["name"]}** (v otázke „{match["input"]}“).'
|
|
for match in matches
|
|
if match["method"] in {"typo", "reordered"}
|
|
]
|
|
return {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"resolved_query": resolved,
|
|
"matches": matches,
|
|
"notices": notices,
|
|
}
|
|
|
|
|
|
def _blocked(query, status, reason, mention, rows):
|
|
candidates = list(
|
|
{
|
|
(row["name"], row.get("source_document")): {
|
|
"name": row["name"],
|
|
"source_document": row.get("source_document"),
|
|
}
|
|
for row in rows
|
|
}.values()
|
|
)
|
|
if status == "ambiguous":
|
|
choices = "; ".join(
|
|
row["name"] + (f' [{row["source_document"]}]' if row.get("source_document") else "")
|
|
for row in candidates
|
|
)
|
|
message = f"Meno „{mention}“ nie je jednoznačné. Ktorú osobu máš na mysli? Možnosti: {choices}. Uveď celé meno"
|
|
message += (
|
|
" a pri rovnakých menách aj cestu profilu."
|
|
if len({r["name"] for r in candidates}) < len(candidates)
|
|
else "."
|
|
)
|
|
else:
|
|
message = f"Meno „{mention}“ sa nepodarilo spoľahlivo priradiť k osobe v dostupných podkladoch. Uveď presné celé meno."
|
|
return {
|
|
"status": status,
|
|
"reason": reason,
|
|
"resolved_query": query,
|
|
"matches": [],
|
|
"notices": [],
|
|
"mention": mention,
|
|
"candidates": candidates,
|
|
"message": message,
|
|
}
|
|
|
|
|
|
def sqlite_people(db_file, *, published_only=True) -> list[dict] | None:
|
|
"""Read profile identities, never infer people from retrieved prose."""
|
|
with closing(sqlite3.connect(db_file.resolve().as_uri() + "?mode=ro", uri=True)) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
columns = {row[1] for row in conn.execute("PRAGMA table_info(documents)")}
|
|
if not {"path", "title"} <= columns:
|
|
# Legacy isolated chunk fixtures have no identity catalog at all.
|
|
# A real documents table with zero profiles remains an empty catalog.
|
|
return None
|
|
where = (
|
|
" WHERE COALESCE(published,1)=1" if published_only and "published" in columns else ""
|
|
)
|
|
fields = "path,title,author" if "author" in columns else "path,title"
|
|
result = []
|
|
for row in conn.execute("SELECT " + fields + " FROM documents" + where):
|
|
profile = re.fullmatch(
|
|
r"(?:pages/)?(?:students/\d{4}/[^/]+|interns/[^/]+)/README\.md", row["path"]
|
|
)
|
|
if row["title"]:
|
|
result.append(
|
|
{
|
|
"kind": "Person" if profile else "Document",
|
|
"name": row["title"],
|
|
"source_document": row["path"],
|
|
}
|
|
)
|
|
if "author" in columns and row["author"]:
|
|
result.append({"kind": "DocumentAuthor", "name": str(row["author"])})
|
|
return result
|