281 lines
16 KiB
Python
281 lines
16 KiB
Python
"""Conservative, catalog-bound person resolution shared by both retrieval engines.
|
|
|
|
Only contiguous full names may be corrected automatically. Fuzzy matches must
|
|
have an exact/inflected anchor, at most one misspelled token and a unique best
|
|
catalog identity. An unresolved explicit subject never means an empty filter.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import defaultdict
|
|
import re
|
|
import sqlite3
|
|
import unicodedata
|
|
from contextlib import closing
|
|
|
|
|
|
def normalize(value: str) -> str:
|
|
text = unicodedata.normalize('NFKD', value).casefold()
|
|
return ' '.join(re.findall(r'[^\W_]+', ''.join(c for c in text if not unicodedata.combining(c))))
|
|
|
|
|
|
def name_token_variants(token: str) -> set[str]:
|
|
variants = {token}
|
|
if len(token) < 3:
|
|
return variants
|
|
irregular = {'peter': ('petra', 'petrovi', 'petrom'),
|
|
'pavol': ('pavla', 'pavlovi', 'pavlom'),
|
|
'frantisek': ('frantiska', 'frantiskovi', 'frantiskom')}
|
|
variants.update(irregular.get(token, ()))
|
|
if token.endswith('ova'):
|
|
variants.update(token[:-1] + s for s in ('ej', 'u', 'ou'))
|
|
elif token.endswith('y'):
|
|
variants.update(token[:-1] + s for s in ('eho', 'emu', 'ym', 'ych'))
|
|
elif token.endswith('a'):
|
|
variants.update(token[:-1] + s for s in ('y', 'e', 'u', 'ou', 'ovi', 'om'))
|
|
elif token.endswith('o'):
|
|
variants.update(token[:-1] + s for s in ('a', 'ovi', 'om'))
|
|
else:
|
|
variants.update(token + s for s in ('a', 'ovi', 'om'))
|
|
return variants
|
|
|
|
|
|
def _distance(a: str, b: str) -> int:
|
|
"""Optimal string alignment distance, including adjacent transpositions."""
|
|
rows = [list(range(len(b) + 1))]
|
|
for i, left in enumerate(a, 1):
|
|
row = [i]
|
|
for j, right in enumerate(b, 1):
|
|
value = min(row[-1] + 1, rows[-1][j] + 1, rows[-1][j-1] + (left != right))
|
|
if i > 1 and j > 1 and left == b[j-2] and a[i-2] == right:
|
|
value = min(value, rows[-2][j-2] + 1)
|
|
row.append(value)
|
|
rows.append(row)
|
|
return rows[-1][-1]
|
|
|
|
|
|
def _score(words: list[str], canonical: list[str]) -> int | None:
|
|
if len(words) != len(canonical):
|
|
return None
|
|
scores = []
|
|
for actual, expected in zip(words, canonical):
|
|
if actual == expected:
|
|
scores.append(0)
|
|
elif actual in name_token_variants(expected):
|
|
scores.append(1)
|
|
else:
|
|
if not actual.isalpha() or not expected.isalpha() or min(len(actual), len(expected)) < 4:
|
|
return None
|
|
cap = 2 if min(len(actual), len(expected)) >= 8 else 1
|
|
forms = name_token_variants(expected)
|
|
distance = min((_distance(actual, form) for form in forms
|
|
if abs(len(actual)-len(form)) <= cap), default=cap+1)
|
|
if distance > cap:
|
|
return None
|
|
scores.append(10 + distance)
|
|
# Never assemble a person from two approximate tokens.
|
|
if sum(s >= 10 for s in scores) > 1 or all(s >= 10 for s in scores):
|
|
return None
|
|
return sum(scores)
|
|
|
|
|
|
_STOP = set('a aj ale alebo ak ako ake aky aku ktore ktori ktora ktory kto kolko ci co obaja obidvaja '
|
|
'ma maju mal mali je su bol bola bolo boli budu bude sa s so pri pre z zo na v od do roku rokoch '
|
|
'praca prace pracu projekt projekty dokument dokumenty nazov nazvy nazv '
|
|
'tag tagy klucove slova slovo temu temy kategoriu kategorie '
|
|
'uved uvedte vypis vyber najdi zisti porovnaj popis opis čomu komu vymenuj pomenuj '
|
|
'vsetci vsetky vsetkych plan stav vysledky problemy metody experimenty '
|
|
'osoba osoby osobu osobe osobach student studenti studentov autora autor '
|
|
'aktualne teraz vedie viedol pokrocil pokrocili kedy prislusne'.split())
|
|
_STOP = {normalize(word) for word in _STOP}
|
|
|
|
|
|
def _generic(word: str) -> bool:
|
|
return word in _STOP or word.startswith(('diplomov', 'bakalarsk', 'dizertacn', 'timov', 'zaverecn',
|
|
'dokument', 'osob', 'student', 'kandidat', 'kategori',
|
|
'zameran', 'ciel', 'vysled', 'priebeh', 'pouz', 'experiment',
|
|
'problem', 'metod', 'riesen', 'system', 'vyhladav', 'inform',
|
|
'studij', 'databaz', 'zdroj', 'prehlad', 'presn', 'postup',
|
|
'hodnot', 'technol', 'zadani', 'uloh', 'spaja', 'evidov', 'veden'))
|
|
|
|
|
|
def _identity(row: dict) -> tuple:
|
|
return (normalize(row['name']), row.get('kind', 'Person'),
|
|
row.get('source_document') or row.get('element_id') or row.get('id') or row['name'])
|
|
|
|
|
|
def resolve_person_names(query: str, catalog: list[dict]) -> dict:
|
|
"""Return an internal canonical query, public matches and fail-closed choices.
|
|
|
|
query itself stays unchanged for HTTP/MCP echo binding. Replacements affect
|
|
name character spans only: work types, years, tags and negations survive.
|
|
"""
|
|
people = list({ _identity(row): row for row in catalog
|
|
if row.get('name') and len(normalize(row['name']).split()) >= 2
|
|
and row.get('kind', 'Person') in {'Person', 'DocumentAuthor'}}.values())
|
|
profile_names = {normalize(row['name']) for row in people if row.get('kind','Person')=='Person'}
|
|
author_lookup = bool(re.search(r'\b(?:autor\w*|dokument\w*|vedie|viedol|veduc\w*|skolitel\w*)\b', normalize(query)))
|
|
people = [row for row in people if row.get('kind')!='DocumentAuthor'
|
|
or author_lookup and normalize(row['name']) not in profile_names]
|
|
tokens = list(re.finditer(r'[^\W_]+', query, re.UNICODE))
|
|
words = [normalize(token[0]) for token in tokens]
|
|
quoted = [(m.start(), m.end()) for m in re.finditer(r'["„“«][^"„“”»]+["“”»]', query)]
|
|
opaque = [(m.start(),m.end()) for m in re.finditer(r'(?:https?://|(?:pages/)?(?:students|interns)/)[^\s]+', query)]
|
|
# A tag/title can itself look like a person's name. Those literals are
|
|
# constraints, not an invitation to correct a student identity.
|
|
literal = [(m.start(1),m.end(1)) for m in re.finditer(
|
|
r'\b(?:tag(?:om|u)?|kľúčov\w*\s+slov\w*|klucov\w*\s+slov\w*|názvom|nazvom)\s+'
|
|
r'([^,;?!\n]+?)(?=\s+(?:a|pri|v\s+roku|z\s+roku|ktor\w*)\b|[,;?!\n]|$)',query,re.I)
|
|
if normalize(m[1]).split()[0] not in {'ma','maju','mal','mali','je','su','bol','boli','uved','vypis'}]
|
|
|
|
def inside_quote(start, end):
|
|
return any(left <= start and end <= right for left, right in [*quoted,*opaque,*literal])
|
|
|
|
spans = defaultdict(list)
|
|
for row in people:
|
|
canonical = normalize(row['name']).split()
|
|
orders = [canonical]
|
|
if len(canonical) >= 2:
|
|
orders.append([canonical[-1], *canonical[:-1]])
|
|
for i in range(len(tokens)-len(canonical)+1):
|
|
start, end = tokens[i].start(), tokens[i+len(canonical)-1].end()
|
|
if inside_quote(start, end):
|
|
continue
|
|
gaps = [query[tokens[j].end():tokens[j+1].start()] for j in range(i, i+len(canonical)-1)]
|
|
if any(not re.fullmatch(r'[\s,]+', gap) for gap in gaps):
|
|
continue
|
|
scores = [(score, order != canonical) for order in orders
|
|
if (score := _score(words[i:i+len(canonical)], order)) is not None]
|
|
if scores:
|
|
score, reversed_order = min(scores)
|
|
spans[(start, end)].append((score, row, reversed_order))
|
|
|
|
selected = []
|
|
for (start, end), options in sorted(spans.items(), key=lambda item: (min(v[0] for v in item[1]),
|
|
-(item[0][1]-item[0][0]), item[0][0])):
|
|
if any(start < right and left < end for left, right, _ in selected):
|
|
continue
|
|
best = min(value[0] for value in options)
|
|
# A complete literal name wins. Otherwise every bounded plausible
|
|
# identity is a choice: the number of inflected tokens is not evidence
|
|
# that one real person is more likely than another.
|
|
choices = [v for v in options if v[0] == 0] if best == 0 else options
|
|
explicit_profiles = [v for v in choices if v[1].get('source_document')
|
|
and v[1]['source_document'] in query]
|
|
if len(explicit_profiles) == 1:
|
|
choices = explicit_profiles
|
|
# A document author and student may share a name without being two
|
|
# student identities. Keep roles separate in graph constraints later.
|
|
by_name = defaultdict(list)
|
|
for value in choices:
|
|
by_name[normalize(value[1]['name'])].append(value)
|
|
duplicate = any(len({_identity(v[1]) for v in group if v[1].get('kind','Person')=='Person'}) > 1
|
|
for group in by_name.values())
|
|
if len(by_name) > 1 or duplicate:
|
|
return _blocked(query, 'ambiguous', 'ambiguous_person', query[start:end], [v[1] for v in choices])
|
|
choices.sort(key=lambda v:v[1].get('kind','Person') != 'Person')
|
|
selected.append((start, end, choices[0]))
|
|
|
|
def covered(start, end):
|
|
return any(left <= start and end <= right for left, right, _ in selected)
|
|
|
|
first_names = {variant for row in people for variant in name_token_variants(normalize(row['name']).split()[0])}
|
|
title_starts = {variant for row in catalog if row.get('kind') not in {'Person','DocumentAuthor',None}
|
|
for key in ('name','title','heading') if row.get(key) and normalize(row[key])
|
|
for variant in name_token_variants(normalize(row[key]).split()[0])}
|
|
first_subject = min((left for left, _, _ in selected), default=len(query))
|
|
list_prefix = normalize(re.split(r'[.!?;\n]', query[:first_subject])[-1])
|
|
explicit_list = bool(re.search(
|
|
r'\b(?:kandidat(?:ov|i)|osob(?:y|ach)?|student(?:i|ov))\s*$', list_prefix))
|
|
# Explicit subjects: full capitalized names; known given names followed by
|
|
# an unknown surname; names after a role, possessive work noun or predicate.
|
|
for i, token in enumerate(tokens):
|
|
if (not any(c.isalpha() for c in words[i]) or _generic(words[i])
|
|
or inside_quote(token.start(), token.end()) or covered(token.start(), token.end())):
|
|
continue
|
|
# A predicate in an earlier sentence cannot introduce this token:
|
|
# "uveď práce. Pridaj zdroje" must not create a person named Pridaj.
|
|
prefix = normalize(re.split(r'[.!?;\n]', query[:token.start()])[-1])
|
|
role = bool(re.search(r'\b(?:osob\w*|student\w*|kandidat\w*|autor(?:a|om)?|ma|mal|robil|riesil|venuje|prace|pracu|projektu)\s*$', prefix))
|
|
# "Pri každej osobe príslušné práce" refers back to the subjects;
|
|
# it does not introduce a person named "príslušné". Do not use this
|
|
# quantified role as evidence of a new name. Independent full-name
|
|
# and known-given-name checks below still detect additional subjects.
|
|
reference = bool(re.search(
|
|
r'\b(?:kazd\w*|jednotliv\w*|vsetk\w*|uveden\w*|vybran\w*|dan\w*|'
|
|
r'obe|oboch|obom|oboma|obidv\w*|tieto|tychto|tymto)\s+'
|
|
r'(?:osob\w*|student\w*|kandidat\w*|autor\w*)\s*$', prefix))
|
|
role = role and not reference
|
|
following = i+1 < len(tokens) and re.fullmatch(r'\s+', query[token.end():tokens[i+1].start()])
|
|
pair = bool(following and not _generic(words[i+1]))
|
|
upper_pair = bool(pair and token[0][0].isupper() and tokens[i+1][0][0].isupper()
|
|
and not token[0].isupper() and not tokens[i+1][0].isupper())
|
|
comparison_subject = bool(selected and pair and re.search(r'\b(?:a|porovnaj)\s*$', prefix))
|
|
# Lower-case unknown entries in an explicit candidate list must still
|
|
# fail closed, even if they follow a comma or a known person.
|
|
list_subject = explicit_list and any(
|
|
right <= token.start() and re.fullmatch(r'\s*(?:,|a)\s*', query[right:token.start()])
|
|
for _, right, _ in selected)
|
|
if not (role or (pair and words[i] in first_names) or upper_pair or comparison_subject or list_subject):
|
|
continue
|
|
end = tokens[i+1].end() if pair else token.end()
|
|
if covered(token.start(), end):
|
|
continue
|
|
mention = query[token.start():end]
|
|
if not pair and words[i] in title_starts:
|
|
continue
|
|
if not pair:
|
|
# Partial names are never fuzzy-expanded. Offer actual catalog choices.
|
|
options = [row for row in people if any(words[i] in name_token_variants(part)
|
|
for part in normalize(row['name']).split())]
|
|
if options:
|
|
return _blocked(query, 'ambiguous', 'incomplete_person_name', mention, options)
|
|
return _blocked(query, 'no_answer', 'unknown_person', mention, [])
|
|
|
|
resolved = query
|
|
matches = []
|
|
for start, end, (score, row, reverse) in sorted(selected, reverse=True):
|
|
resolved = resolved[:start] + row['name'] + resolved[end:]
|
|
matches.append({'input': query[start:end], 'name': row['name'],
|
|
'method': 'typo' if score >= 10 else 'reordered' if reverse else 'variant',
|
|
'source_document': row.get('source_document')})
|
|
matches.reverse()
|
|
notices = [f'Rozpoznané meno: **{match["name"]}** (v otázke „{match["input"]}“).'
|
|
for match in matches if match['method'] in {'typo', 'reordered'}]
|
|
return {'status': 'ok', 'reason': None, 'resolved_query': resolved, 'matches': matches, 'notices': notices}
|
|
|
|
|
|
def _blocked(query, status, reason, mention, rows):
|
|
candidates = list({(row['name'],row.get('source_document')):
|
|
{'name':row['name'],'source_document':row.get('source_document')} for row in rows}.values())
|
|
if status == 'ambiguous':
|
|
choices = '; '.join(row['name'] + (f' [{row["source_document"]}]' if row.get('source_document') else '')
|
|
for row in candidates)
|
|
message = f'Meno „{mention}“ nie je jednoznačné. Ktorú osobu máš na mysli? Možnosti: {choices}. Uveď celé meno'
|
|
message += ' a pri rovnakých menách aj cestu profilu.' if len({r['name'] for r in candidates}) < len(candidates) else '.'
|
|
else:
|
|
message = f'Meno „{mention}“ sa nepodarilo spoľahlivo priradiť k osobe v dostupných podkladoch. Uveď presné celé meno.'
|
|
return {'status':status,'reason':reason,'resolved_query':query,'matches':[], 'notices':[],
|
|
'mention':mention,'candidates':candidates,'message':message}
|
|
|
|
|
|
def sqlite_people(db_file, *, published_only=True) -> list[dict] | None:
|
|
"""Read profile identities, never infer people from retrieved prose."""
|
|
with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro', uri=True)) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
columns = {row[1] for row in conn.execute('PRAGMA table_info(documents)')}
|
|
if not {'path','title'} <= columns:
|
|
# Legacy isolated chunk fixtures have no identity catalog at all.
|
|
# A real documents table with zero profiles remains an empty catalog.
|
|
return None
|
|
where = ' WHERE COALESCE(published,1)=1' if published_only and 'published' in columns else ''
|
|
fields = 'path,title,author' if 'author' in columns else 'path,title'
|
|
result = []
|
|
for row in conn.execute('SELECT '+fields+' FROM documents'+where):
|
|
profile = re.fullmatch(r'(?:pages/)?(?:students/\d{4}/[^/]+|interns/[^/]+)/README\.md', row['path'])
|
|
if row['title']:
|
|
result.append({'kind':'Person' if profile else 'Document','name':row['title'],
|
|
'source_document':row['path']})
|
|
if 'author' in columns and row['author']:
|
|
result.append({'kind':'DocumentAuthor','name':str(row['author'])})
|
|
return result
|