715 lines
38 KiB
Python
715 lines
38 KiB
Python
"""Verified answers from CRZP metadata (/crzp).
|
||
|
||
Flow: question -> CrzpQuery -> CRZP searches (years and work types filtered by
|
||
CRZP itself, cached) -> rows filtered by person role, school and verified
|
||
workplace -> answer by intent. A topic is also searched under up to two
|
||
alternative terms (synonyms) and the answer names every searched term. Every
|
||
listed fact is copied from a parsed CRZP record and every work links to its
|
||
CRZP permalink. The model only (1) proposes base forms of names/topics and
|
||
alternative terms, checked by code, and (2) summarizes an abstract; the
|
||
summary is shown only after a separate check that it is supported by the
|
||
abstract, otherwise the abstract itself is shown.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import threading
|
||
import time
|
||
from collections import Counter, OrderedDict
|
||
|
||
from tools.answers.delivery_contract import fingerprint
|
||
from tools.crzp import fetch
|
||
from tools.crzp.answering import context
|
||
from tools.crzp.answering.names import name_variants
|
||
from tools.crzp.answering.terms import is_reviewed
|
||
from tools.crzp.answering.understand import SCHOOLS, CrzpQuery, in_question, stem_match, understand
|
||
from tools.crzp.answering.units import UNITS, in_unit, unit_label
|
||
from tools.crzp.source.client import CrzpUnavailable
|
||
from tools.crzp.source.parse import WORK_TYPE_LABELS, display_name, fold
|
||
from tools.crzp.storage import graph
|
||
|
||
ENGINE = "crzp"
|
||
LIST_LIMIT = 30
|
||
ROLE_FIELD = {"author": "authors", "supervisor": "supervisors", "opponent": "opponents"}
|
||
ROLE_LABEL = {"author": "autor", "supervisor": "školiteľ", "opponent": "oponent"}
|
||
ROLE_PLURAL = {"author": "Autori", "supervisor": "Školitelia", "opponent": "Oponenti"}
|
||
FACET = {"author": "Autor", "supervisor": "Školiteľ"}
|
||
_cache: OrderedDict = OrderedDict()
|
||
_cache_lock = threading.Lock()
|
||
CACHE_SECONDS = 600
|
||
|
||
|
||
# ------------------------------------------------------------------ matching
|
||
|
||
def name_matches(query_name: str, crzp_name: str) -> bool:
|
||
"""Every word of the asked name is a word of the CRZP name (up to inflection)."""
|
||
words = re.findall(r"[^\W\d_]+", crzp_name.split("(")[0])
|
||
asked = re.findall(r"[^\W\d_]+", query_name)
|
||
return bool(asked) and all(any(stem_match(a, w) for w in words) for a in asked)
|
||
|
||
|
||
def people_in_role(row: dict, name: str, role: str) -> list[str]:
|
||
fields = [ROLE_FIELD[role]] if role in ROLE_FIELD else list(ROLE_FIELD.values())
|
||
return [p for f in fields for p in row.get(f, []) if name_matches(name, p)]
|
||
|
||
|
||
def keep(row: dict, query: CrzpQuery) -> bool:
|
||
"""Conditions CRZP cannot filter (roles, school, workplace) plus a repeat of
|
||
the year/type filters (a year list may have gaps inside the searched range)."""
|
||
if query.sids and row.get("sid") not in query.sids:
|
||
return False # follow-up about chosen works of the previous answer
|
||
if query.school_name and university(row) != query.school_name:
|
||
return False # the chosen namesake
|
||
if any(not people_in_role(row, p.name, p.role) for p in query.people):
|
||
return False
|
||
if query.years and row.get("year") not in query.years:
|
||
return False
|
||
if query.work_types and row.get("work_type") not in query.work_types:
|
||
return False
|
||
if query.school and SCHOOLS[query.school][1] not in fold(university(row)): # "Slovenská technická univ."
|
||
return False
|
||
if query.unit and not in_unit(row, query.unit):
|
||
return False
|
||
return True
|
||
|
||
|
||
def filters(query: CrzpQuery) -> dict:
|
||
"""Filters CRZP applies itself: a range of submission years and work types."""
|
||
return {"years": (min(query.years), max(query.years)) if query.years else None,
|
||
"work_types": list(query.work_types) or None}
|
||
|
||
|
||
def suffixes(query: CrzpQuery) -> list[str]:
|
||
"""What a topic search appends for a school or workplace: the school name, or every
|
||
form CRZP writes the workplace in (code "104005" and, in old records, the department name)."""
|
||
if query.unit:
|
||
return [" " + text for text in UNITS[query.unit]["search"]]
|
||
return [" " + SCHOOLS[query.school][0]] if query.school else [""]
|
||
|
||
|
||
def strip_suffix(query: CrzpQuery, text: str) -> str:
|
||
return next((text[: -len(s)] for s in suffixes(query) if s and text.endswith(s) and text != s.strip()), text)
|
||
|
||
|
||
def search_groups(query: CrzpQuery) -> list[list[str]]:
|
||
"""Search texts in order of preference; the texts of one group are searched
|
||
together and their rows merged (a topic with its alternative terms). A
|
||
verified workplace is searched by its code (CRZP indexes it, "104005") and
|
||
by its old written name."""
|
||
ends = suffixes(query)
|
||
each = lambda terms: [t + end for t in terms for end in ends]
|
||
if query.people:
|
||
names = [query.people[0].name] if query.from_model else name_variants(query.people[0].name)
|
||
title = " " + query.title if query.title else ""
|
||
return ([[f"{n} {query.topics[0]}{title}"] for n in names if query.topics] + [[n + title] for n in names])
|
||
if query.title:
|
||
return [[query.title]]
|
||
if not query.topics:
|
||
if query.unit:
|
||
return [list(UNITS[query.unit]["search"]), [SCHOOLS[query.school][0]]]
|
||
return [[SCHOOLS[query.school][0]]] if query.school else []
|
||
main = " ".join(query.topics[:2])
|
||
if query.phrases and not query.from_model: # rules: the likely written phrase first
|
||
groups = [each([query.phrases[0], *query.alternatives])] + [each([p]) for p in query.phrases[1:]]
|
||
groups.append(each([main]))
|
||
else:
|
||
groups = [each([main, *query.alternatives])]
|
||
if len(query.topics) > 1: # two topic words did not occur together: the main one alone
|
||
groups.append(each([query.topics[0]]))
|
||
groups += [each([t]) for t in query.surface[:1]] # words as written (rules without the model)
|
||
if ends != [""]:
|
||
groups += [[t] for t in [main, *query.alternatives]]
|
||
unique, seen = [], set()
|
||
for group in groups:
|
||
group = [t for t in dict.fromkeys(group) if t.strip() and t not in seen]
|
||
seen.update(group)
|
||
if group:
|
||
unique.append(group)
|
||
return unique
|
||
|
||
|
||
def _exact(name: str, crzp_name: str) -> bool:
|
||
"""Same words including diacritics, up to inflection ("Jána" is "Ján", not "Jan")."""
|
||
words = re.findall(r"[^\W\d_]+", crzp_name)
|
||
return all(any(stem_match(a, w, diacritics=True) for w in words) for a in re.findall(r"[^\W\d_]+", name))
|
||
|
||
|
||
def prefer_exact(rows: list[dict], query: CrzpQuery) -> list[dict]:
|
||
"""A name typed with diacritics keeps only the exact spellings when CRZP has them."""
|
||
for person in query.people:
|
||
typed = person.typed or person.name
|
||
if fold(typed) == typed.lower():
|
||
continue # typed without diacritics: every spelling matches
|
||
exact = [r for r in rows if any(_exact(typed, n) for n in people_in_role(r, person.name, person.role))]
|
||
if exact:
|
||
rows = exact
|
||
return rows
|
||
|
||
|
||
def identities(rows: list[dict], person) -> list[dict]:
|
||
"""The asked name split into probable persons: the same written name (with diacritics)
|
||
at the same university is one person; another spelling or university is another one.
|
||
CRZP has no person identifiers, so this is the most it can tell."""
|
||
groups: dict = {}
|
||
for row in rows:
|
||
for name in people_in_role(row, person.name, person.role):
|
||
key = (display_name(name), university(row))
|
||
group = groups.setdefault(key, {"name": key[0], "school": key[1], "count": 0, "years": set(),
|
||
"roles": Counter()})
|
||
group["count"] += 1
|
||
group["years"].add(row.get("year"))
|
||
group["roles"].update(role for role, field in ROLE_FIELD.items() if name in row.get(field, []))
|
||
return sorted(groups.values(), key=lambda g: (-g["count"], -max((y or 0) for y in g["years"]), g["name"], g["school"]))
|
||
|
||
|
||
def _identity_line(group: dict) -> str:
|
||
years = sorted(y for y in group["years"] if y)
|
||
span = (f"{years[0]}–{years[-1]}" if len(years) > 1 else str(years[0])) if years else "rok neuvedený"
|
||
roles = ", ".join(ROLE_LABEL[r] for r, _ in group["roles"].most_common())
|
||
works = f"{group['count']} {'práca' if group['count'] == 1 else 'práce' if group['count'] < 5 else 'prác'}"
|
||
return f"{group['name']} — {group['school']} ({roles}; {span}; {works})"
|
||
|
||
|
||
def namesakes(rows: list[dict], query: CrzpQuery) -> str:
|
||
"""Notice when the asked name belongs to several probable CRZP persons."""
|
||
for person in query.people:
|
||
groups = identities(rows, person)
|
||
if len(groups) > 1:
|
||
return ("\n\nUpozornenie: menu zodpovedá v CRZP viac pravdepodobných osôb (CRZP ich nerozlišuje "
|
||
"identifikátorom):\n" + "\n".join("- " + _identity_line(g) for g in groups[:6]) +
|
||
"\nSpresni školu alebo rok.")
|
||
return ""
|
||
|
||
|
||
def clarification(rows: list[dict], query: CrzpQuery) -> tuple[str, list[dict]] | None:
|
||
"""Count, summary, keywords, relations and similar works of one person must not mix
|
||
namesakes: ask which person is meant (a list shows all with a notice instead)."""
|
||
if not query.people or query.intent in ("list", "schools") or query.school_name or query.sids:
|
||
return None
|
||
groups = identities(rows, query.people[0])
|
||
if len(groups) < 2:
|
||
return None
|
||
groups = groups[:8]
|
||
text = ("Menu „" + query.people[0].name + "“ zodpovedá v CRZP viac pravdepodobných osôb (CRZP ich nerozlišuje "
|
||
"identifikátorom). Ktorú myslíš?\n\n" + "\n".join(f"{i}. {_identity_line(g)}" for i, g in
|
||
enumerate(groups, 1)) +
|
||
"\n\nOdpovedz číslom, školou alebo celým menom.")
|
||
return text, [{"name": g["name"], "school": g["school"]} for g in groups]
|
||
|
||
|
||
def _merge(results: list[dict]) -> dict:
|
||
rows, seen = [], set()
|
||
for result in results:
|
||
for row in result["rows"]:
|
||
key = row.get("sid") or (fold(row["title"]), row.get("year"), tuple(row.get("authors", [])))
|
||
if key not in seen:
|
||
seen.add(key)
|
||
rows.append(row)
|
||
return {"texts": [r["query"] for r in results], "totals": [r["total"] for r in results],
|
||
"total": results[0]["total"] if len(results) == 1 else sum(r["total"] for r in results),
|
||
"facets": results[0]["facets"], "rows": rows, "complete": all(r["complete"] for r in results),
|
||
"incomplete": [(r["query"], r["total"], len(r["rows"])) for r in results if not r["complete"]]}
|
||
|
||
|
||
def _unverified(query: CrzpQuery, text: str) -> str | None:
|
||
"""The model-proposed alternative term the search text stands for (not in the reviewed table)."""
|
||
term = strip_suffix(query, text)
|
||
for alternative in query.alternatives:
|
||
if fold(alternative) == fold(term) and query.topics and not is_reviewed(alternative, query.topic_key()):
|
||
return alternative
|
||
return None
|
||
|
||
|
||
def _used_with_topic(term: str, rows: list[dict]) -> bool:
|
||
"""CRZP itself writes the term in a work found for the topic (title or keywords)."""
|
||
return any(in_question(term, r["title"] + " " + " ".join(r.get("keywords", []))) for r in rows)
|
||
|
||
|
||
AGGREGATES = ("count", "keywords", "schools", "relation")
|
||
|
||
|
||
def find(query: CrzpQuery, *, links: bool = True) -> tuple[dict | None, list[dict]]:
|
||
"""First search group whose rows satisfy all conditions; (merged result, rows).
|
||
A model-proposed alternative term is searched only if CRZP uses it in a work
|
||
already found for the topic, or if nothing was found yet."""
|
||
last, seen = None, []
|
||
query.searched = []
|
||
# Counts and summaries read up to DEEP_ROWS rows (no permalinks needed); a person's
|
||
# works are filtered by role in code, so all of them are read as well.
|
||
depth = fetch.DEEP_ROWS if query.intent in AGGREGATES or query.people else fetch.LINKED_ROWS
|
||
for group in search_groups(query):
|
||
results, verified_rows, unverified = [], 0, []
|
||
for text in group:
|
||
term = _unverified(query, text)
|
||
if term and seen and not _used_with_topic(term, seen):
|
||
continue # "rozložená kniha" for blockchain: CRZP does not use it, no search
|
||
results.append(fetch.search(text, links=links, depth=depth, **filters(query)))
|
||
query.searched.append(text)
|
||
seen += results[-1]["rows"]
|
||
if term:
|
||
unverified.append(term)
|
||
else:
|
||
verified_rows += len(results[-1]["rows"])
|
||
if not results:
|
||
continue
|
||
found = _merge(results)
|
||
rows = prefer_exact([r for r in found["rows"] if keep(r, query)], query)
|
||
last = (found, rows)
|
||
if rows and unverified and not verified_rows:
|
||
query.unverified_only = unverified # the topic itself found nothing: say so in the answer
|
||
if rows:
|
||
if query.people: # show the name as CRZP writes it ("hladek" -> "Daniel Hládek") when it is one person
|
||
names = Counter(n for r in rows for n in people_in_role(r, query.people[0].name, query.people[0].role))
|
||
if len(names) == 1 or not query.from_model:
|
||
query.people[0].name = display_name(names.most_common(1)[0][0])
|
||
return last
|
||
if last and query.people and query.people[0].role == "author":
|
||
# "práce X" may mean the works X supervised or reviewed: X is no author in the read rows.
|
||
relaxed = query.model_copy(deep=True)
|
||
relaxed.people[0].role = "any"
|
||
rows = prefer_exact([r for r in last[0]["rows"] if keep(r, relaxed)], relaxed)
|
||
if rows:
|
||
query.people[0].role = "any"
|
||
query.relaxed_author = True
|
||
return last[0], rows
|
||
return last or (None, [])
|
||
|
||
|
||
def ranked(question: str, query: CrzpQuery, rows: list[dict]) -> list[dict]:
|
||
"""Topic questions: cross-encoder relevance over title + CRZP keywords; else newest first."""
|
||
rows = sorted(rows, key=lambda r: (-(r.get("year") or 0), fold(r["title"])))
|
||
if query.topics and len(rows) > 1:
|
||
from tools.retrieval import rerank
|
||
# The searched terms tell the cross-encoder what an abbreviation means ("LLM jazykový model GPT").
|
||
terms = [t for t in [*query.topics, *query.alternatives] if fold(t) not in fold(question)]
|
||
rows = rerank.order(" ".join([question, *terms]), rows, key=lambda r: r["title"] + ". " + ", ".join(r.get("keywords", [])))
|
||
return rows
|
||
|
||
|
||
# ------------------------------------------------------------------ rendering
|
||
|
||
def _names(values: list[str]) -> str:
|
||
return ", ".join(display_name(v) for v in values) or "neuvedené"
|
||
|
||
|
||
def _kind(row: dict) -> str:
|
||
return WORK_TYPE_LABELS.get(row.get("work_type"), "práca")
|
||
|
||
|
||
def _link(row: dict) -> str:
|
||
return f"[CRZP]({row['url']})" if row.get("url") else "záznam bez trvalého odkazu"
|
||
|
||
|
||
def school(row: dict) -> str:
|
||
unit = unit_label(row)
|
||
return (row.get("school") or "škola neuvedená") + (f", {unit}" if unit else "")
|
||
|
||
|
||
def work_block(row: dict, detail: dict | None = None) -> str:
|
||
record = detail or row
|
||
head = f"**{record['title']}** — {_kind(record)}, {record.get('year') or 'rok neuvedený'}, {school(record)}"
|
||
lines = [head, f"- Autor: {_names(record.get('authors', []))}; školiteľ: {_names(record.get('supervisors', []))}; "
|
||
f"oponent: {_names(record.get('opponents', []))}"]
|
||
if detail and detail.get("consultants"):
|
||
lines.append(f"- Konzultant: {_names(detail['consultants'])}")
|
||
if record.get("keywords"):
|
||
lines.append("- Kľúčové slová: " + ", ".join(record["keywords"]))
|
||
if detail:
|
||
extra = [f"{detail['pages']} strán" if detail.get("pages") else "",
|
||
f"jazyk: {detail['language']}" if detail.get("language") else "",
|
||
f"odbor: {detail['field_of_study']}" if detail.get("field_of_study") else ""]
|
||
if any(extra):
|
||
lines.append("- " + "; ".join(e for e in extra if e))
|
||
lines.append("- Zdroj: " + _link(record))
|
||
return "\n".join(lines)
|
||
|
||
|
||
def work_line(row: dict) -> str:
|
||
return (f"- **{row['title']}** ({_kind(row)}, {row.get('year') or '?'}, {school(row)}) — "
|
||
f"{_names(row.get('authors', []))} · {_link(row)}")
|
||
|
||
|
||
def describe(query: CrzpQuery) -> str:
|
||
parts = []
|
||
for person in query.people:
|
||
parts.append(f"{ROLE_LABEL.get(person.role, 'osoba')} {person.name}" if person.role != "any" else person.name)
|
||
if query.title:
|
||
parts.append(f"názov „{query.title}“")
|
||
if query.topics and not query.people:
|
||
parts.append("téma „" + " ".join(query.topics[:2]) + "“")
|
||
if query.work_types:
|
||
parts.append(", ".join(WORK_TYPE_LABELS[t] for t in query.work_types))
|
||
if query.years:
|
||
parts.append("rok " + ", ".join(map(str, query.years)))
|
||
if query.sids:
|
||
parts.append("vybraná práca z predchádzajúcej odpovede" if len(query.sids) == 1
|
||
else "práce z predchádzajúcej odpovede")
|
||
if query.unit:
|
||
parts.append(UNITS[query.unit]["label"] + " " + SCHOOLS[query.school][0])
|
||
elif query.school:
|
||
parts.append(SCHOOLS[query.school][0])
|
||
return "; ".join(parts)
|
||
|
||
|
||
# ------------------------------------------------------------------ model steps
|
||
|
||
SUMMARY_PROMPT = ("Na základe abstraktu a kľúčových slov záverečnej práce napíš po slovensky 2 až 3 vety: o čom práca "
|
||
"je a čo v nej autor urobil. Môžeš preformulovať, ale použi iba informácie z abstraktu; nepridávaj "
|
||
"výsledky, nástroje ani hodnotenia, ktoré v ňom nie sú. Bez úvodu a bez nadpisu.")
|
||
CHECK_PROMPT = ("Over, či je každé tvrdenie zhrnutia podložené abstraktom (preformulovanie je v poriadku, nový fakt nie). "
|
||
"Odpovedz iba JSON {\"supported\": true} alebo {\"supported\": false}.")
|
||
|
||
|
||
def _ask(system: str, payload: dict, complete=None, max_tokens: int = 400) -> str | None:
|
||
from tools.answers.model_client import complete_with_configured_model
|
||
try:
|
||
response = (complete or complete_with_configured_model)(
|
||
[{"role": "system", "content": system}, {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}],
|
||
max_tokens=max_tokens)
|
||
return response["choices"][0]["message"]["content"].strip()
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def summary(detail: dict, complete=None) -> tuple[str, str]:
|
||
"""("Zhrnutie", text) after a passed check, else ("Abstrakt", abstract)."""
|
||
abstract = detail.get("abstract") or detail.get("abstract_secondary") or ""
|
||
if not abstract:
|
||
return "Abstrakt", "CRZP pri tejto práci abstrakt neuvádza."
|
||
text = _ask(SUMMARY_PROMPT, {"title": detail["title"], "abstract": abstract, "keywords": detail.get("keywords", [])},
|
||
complete)
|
||
if text and len(text) < 1200:
|
||
verdict = _ask(CHECK_PROMPT, {"abstract": abstract, "summary": text}, complete, max_tokens=50)
|
||
if verdict and re.search(r'"supported"\s*:\s*true', verdict):
|
||
return "Zhrnutie podľa abstraktu", text
|
||
return "Abstrakt", abstract if len(abstract) <= 1500 else abstract[:1500].rsplit(" ", 1)[0] + " …"
|
||
|
||
|
||
# ------------------------------------------------------------------ answers by intent
|
||
|
||
def _list(question, query, found, rows) -> tuple[str, list[dict]]:
|
||
rows = ranked(question, query, rows)
|
||
header = f"V CRZP som našiel {len(rows)} {'prácu' if len(rows) == 1 else 'práce' if len(rows) < 5 else 'prác'} ({describe(query)}):"
|
||
if len(rows) <= 5:
|
||
return header + "\n\n" + "\n\n".join(work_block(r) for r in rows), rows
|
||
shown = rows[:LIST_LIMIT]
|
||
text = header + "\n\n" + "\n".join(work_line(r) for r in shown)
|
||
if len(rows) > LIST_LIMIT:
|
||
text += f"\n\nZobrazených je {LIST_LIMIT} z {len(rows)}; spresni rok, typ práce alebo tému."
|
||
return text, shown
|
||
|
||
|
||
def _count(question, query, found, rows) -> tuple[str, list[dict]]:
|
||
count, note = len(rows), ""
|
||
person = query.people[0] if query.people else None
|
||
if not found["complete"]:
|
||
facet = FACET.get(person.role) if person else None
|
||
values = [f["count"] for f in found["facets"].get(facet, []) if name_matches(person.name, f["label"])] if facet else []
|
||
# The facet counts the whole CRZP result, which already respects the year and type filters.
|
||
school_only = (query.school and not query.people and not query.topics and not query.title
|
||
and found["texts"] == (UNITS[query.unit]["search"] if query.unit else [SCHOOLS[query.school][0]]))
|
||
if values and len(found["texts"]) == 1 and not (query.school or query.unit or len(query.people) > 1):
|
||
count, note = sum(values), " (súhrnný počet z CRZP)"
|
||
elif school_only and len(rows) == len(found["rows"]):
|
||
count = found["total"]
|
||
several = len(found["texts"]) > 1
|
||
note = (f" (súhrn CRZP pre hľadanie {'pracoviska (kód a starší zápis názvom)' if several else 'kódu pracoviska' if query.unit else 'názvu školy'} "
|
||
f"s filtrom roku a typu práce; všetkých {len(rows)} prehľadaných záznamov bolo z tohto pracoviska)")
|
||
else:
|
||
note = f" (aspoň; prehľadaných {len(found['rows'])} z {found['total']} výsledkov hľadania)"
|
||
types = Counter(_kind(r) for r in rows)
|
||
years = Counter(r.get("year") for r in rows if r.get("year"))
|
||
text = f"Počet prác v CRZP ({describe(query)}): **{count}**{note}."
|
||
if rows:
|
||
text += "\n\n- Podľa typu: " + ", ".join(f"{k} {v}" for k, v in types.most_common())
|
||
text += "\n- Podľa roku: " + ", ".join(f"{y}: {n}" for y, n in sorted(years.items(), reverse=True))
|
||
return text, rows
|
||
|
||
|
||
def _about(question, query, found, rows, complete=None) -> tuple[str, list[dict]]:
|
||
candidates = rows
|
||
unique, seen = [], set()
|
||
for row in ranked(question, query, rows): # CRZP sometimes holds one work twice (two records)
|
||
key = (fold(row["title"]).rstrip(" ."), row.get("year"), tuple(row.get("authors", [])))
|
||
if key not in seen:
|
||
seen.add(key)
|
||
unique.append(row)
|
||
rows = unique[:2]
|
||
details = [fetch.detail(row["sid"]) if row.get("sid") else None for row in rows]
|
||
# The model summaries (and their checks) of the two works run at the same time.
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
summaries = list(pool.map(lambda d: summary(d, complete) if d else None, details))
|
||
blocks = []
|
||
for row, detail, done in zip(rows, details, summaries):
|
||
if detail is None:
|
||
blocks.append(work_block(row) + "\n\nDetail záznamu (abstrakt) nie je dostupný bez trvalého odkazu.")
|
||
continue
|
||
label, text = done
|
||
blocks.append(work_block(row, detail) + f"\n\n{label}: {text}")
|
||
more = len(candidates) > len(rows)
|
||
tail = "\n\nZodpovedá viac prác; zobrazené sú najrelevantnejšie. Spresni typ práce alebo rok." if more else ""
|
||
return "\n\n".join(blocks) + tail, rows
|
||
|
||
|
||
def _keywords(question, query, found, rows) -> tuple[str, list[dict]]:
|
||
rows = ranked(question, query, rows)
|
||
if len(rows) <= 3:
|
||
lines = [f"**{r['title']}** ({_kind(r)}, {r.get('year') or '?'}): " +
|
||
(", ".join(r["keywords"]) if r.get("keywords") else "CRZP kľúčové slová neuvádza") + f" · {_link(r)}"
|
||
for r in rows]
|
||
return "Kľúčové slová (tagy) v CRZP:\n\n" + "\n\n".join(lines), rows
|
||
counts = Counter(k.lower() for r in rows for k in r.get("keywords", []))
|
||
lines = [f"- {k} ({n})" for k, n in counts.most_common(20)]
|
||
return (f"Najčastejšie kľúčové slová (tagy) v {len(rows)} prácach ({describe(query)}):\n\n" + "\n".join(lines),
|
||
rows)
|
||
|
||
|
||
def _relation(question, query, found, rows) -> tuple[str, list[dict]]:
|
||
target = query.target_role
|
||
if query.title: # the asked work itself, if CRZP has that exact title
|
||
exact = [r for r in rows if fold(r["title"]).rstrip(" .") == fold(query.title).rstrip(" .")]
|
||
rows = exact or rows
|
||
if len(rows) <= 3:
|
||
lines = [f"- **{r['title']}** ({_kind(r)}, {r.get('year') or '?'}): {ROLE_LABEL[target]} "
|
||
f"{_names(r.get(ROLE_FIELD[target], []))} · {_link(r)}" for r in rows]
|
||
return f"{ROLE_PLURAL[target]} prác ({describe(query)}):\n\n" + "\n".join(lines), rows
|
||
plural = lambda c: f"{c} {'práca' if c == 1 else 'práce' if c < 5 else 'prác'}"
|
||
if not query.people: # "Kto viedol práce o X?": the target role over the found works
|
||
counts = Counter(n for r in rows for n in r.get(ROLE_FIELD[target], []))
|
||
lines = [f"- {display_name(n)} ({plural(c)})" for n, c in sorted(counts.items(), key=lambda i: (-i[1], i[0]))[:15]]
|
||
return (f"{ROLE_PLURAL[target]} prác ({describe(query)}; {len(rows)} prác v CRZP):\n\n" +
|
||
("\n".join(lines) or "- CRZP ich neuvádza")), ranked(question, query, rows)[:10]
|
||
person = query.people[0]
|
||
source = person.role if person.role in ROLE_FIELD else max(
|
||
ROLE_FIELD, key=lambda role: sum(bool(people_in_role(r, person.name, role)) for r in rows))
|
||
names = sorted({n for r in rows for n in people_in_role(r, person.name, source)})
|
||
pairs = graph.relation(rows, names, source, target)
|
||
lines = [f"- {display_name(n)} ({plural(c)})" for n, c in pairs[:15]]
|
||
conditions = describe(query.model_copy(update={"people": []}))
|
||
text = (f"{ROLE_PLURAL[target]} prác, pri ktorých je {display_name(names[0]) if names else person.name} "
|
||
f"{ROLE_LABEL[source]} ({conditions + '; ' if conditions else ''}{len(rows)} prác v CRZP):\n\n" + ("\n".join(lines) or "- CRZP ich neuvádza"))
|
||
return text, rows
|
||
|
||
|
||
def university(row: dict) -> str:
|
||
"""The school without its faculty/department ("Slovenská technická univ. v Bratislave FEI ÚMIKT")."""
|
||
name = row.get("school") or "škola neuvedená"
|
||
folded = fold(name).replace("univ.", "univerzita").replace("p.j. safarika", "pavla jozefa safarika")
|
||
folded = folded.replace("a.dubceka", "alexandra dubceka")
|
||
known = next((full for full, part in SCHOOLS.values() if part in folded), None)
|
||
if known:
|
||
return known
|
||
town = re.match(r"^(.*?\bv[eo]?\s+[^\W\d_]+)", name) # "… univerzita v Trnave FF KSJ" -> up to the town
|
||
return town.group(1) if town else name
|
||
|
||
|
||
def _faculty(raw: str, name: str) -> str:
|
||
"""The part of a raw CRZP school after the university's town ("… v Bratislave FEI ÚMIKT" -> "FEI ÚMIKT")."""
|
||
town = fold(name.split()[-1])
|
||
at = fold(raw).find(town)
|
||
return raw[at + len(town):] if at >= 0 else ""
|
||
|
||
|
||
def _schools(question, query, found, rows) -> tuple[str, list[dict]]:
|
||
""""Z akých škôl boli tí študenti?": schools of the found works, with their workplaces."""
|
||
plural = lambda c: f"{c} {'práca' if c == 1 else 'práce' if c < 5 else 'prác'}"
|
||
by_school: dict[str, Counter] = {}
|
||
for r in rows:
|
||
unit = unit_label(r) if university(r) == r.get("school") else _faculty(r.get("school") or "", university(r))
|
||
by_school.setdefault(university(r), Counter())[unit.strip(" ,") or ""] += 1
|
||
order = sorted(by_school.items(), key=lambda i: (-sum(i[1].values()), i[0]))
|
||
lines = []
|
||
for name, units in order[:20]:
|
||
parts = [f"{u} {c}" for u, c in units.most_common() if u][:4]
|
||
lines.append(f"- {name} ({plural(sum(units.values()))}" + (": " + "; ".join(parts) if parts else "") + ")")
|
||
more = f"\n- … a ďalšie ({len(order) - 20})" if len(order) > 20 else ""
|
||
return (f"Školy prác v CRZP ({describe(query)}; {plural(len(rows))}):\n\n"
|
||
+ "\n".join(lines) + more), ranked(question, query, rows)[:10]
|
||
|
||
|
||
GENERIC_TAGS = {fold(w.strip()) for w in """detekcia, analýza, systém, aplikácia, návrh, implementácia, model, modelovanie,
|
||
metóda, metódy, dáta, údaje, databáza, simulácia, optimalizácia, riadenie, meranie, testovanie, vývoj, softvér, hardvér,
|
||
internet, web, python, java, c#, matlab, korpus, korpus dát, dataset, klasifikácia, algoritmus, algoritmy""".split(",")}
|
||
|
||
|
||
def specific_tags(work: dict) -> list[str]:
|
||
"""The work's keywords most likely to find related works: multi-word and non-generic first."""
|
||
tags = [k for k in work.get("keywords", []) if fold(k) not in GENERIC_TAGS]
|
||
return sorted(tags, key=lambda k: (-min(len(k.split()), 3), -len(k)))
|
||
|
||
|
||
def _similar(question, query, found, rows) -> tuple[str, list[dict]]:
|
||
work = ranked(question, query, rows)[0]
|
||
if query.title:
|
||
work = next((r for r in rows if fold(r["title"]).rstrip(" .") == fold(query.title).rstrip(" .")), work)
|
||
searched, candidates = [], []
|
||
# Most specific tags first; a tag only this work has ("rasa forms") gives nothing,
|
||
# so the next one is searched until five related works are found (at most four searches).
|
||
for term in specific_tags(work)[:4]:
|
||
try:
|
||
candidates += fetch.search(term)["rows"]
|
||
except CrzpUnavailable:
|
||
break
|
||
searched.append(term)
|
||
if len(graph.shared_keywords(work, candidates)) >= 5:
|
||
break
|
||
pairs = graph.shared_keywords(work, candidates)
|
||
if len(pairs) > 1: # equal numbers of shared tags: the closer title and tags first (cross-encoder)
|
||
from tools.retrieval import rerank
|
||
profile = work["title"] + ". " + ", ".join(work.get("keywords", []))
|
||
scores = rerank.scores(profile, [c["title"] + ". " + ", ".join(c.get("keywords", [])) for c, _ in pairs]) or [0] * len(pairs)
|
||
pairs = [p for _, p in sorted(zip(scores, pairs), key=lambda item: (-len(item[1][1]), -item[0]))]
|
||
pairs = pairs[:5]
|
||
if not pairs:
|
||
return (f"K práci **{work['title']}** som medzi prácami s jej kľúčovými slovami nenašiel inú so spoločnými "
|
||
f"kľúčovými slovami. · {_link(work)}"), [work]
|
||
lines = [f"- **{c['title']}** ({_kind(c)}, {c.get('year') or '?'}, {school(c)}) — spoločné: "
|
||
f"{', '.join(shared)} · {_link(c)}" for c, shared in pairs]
|
||
text = (f"Práce so spoločnými kľúčovými slovami CRZP ako **{work['title']}** ({_link(work)}); porovnané s prácami "
|
||
f"nájdenými podľa kľúčových slov „{'“, „'.join(searched)}“:\n\n" + "\n".join(lines))
|
||
return text, [work] + [c for c, _ in pairs]
|
||
|
||
|
||
# ------------------------------------------------------------------ entry point
|
||
|
||
def _result(question: str, status: str, text: str, *, reason=None, rows=(), complete=None, truncated=False,
|
||
reasons=(), count=None) -> dict:
|
||
urls = list(dict.fromkeys(r["url"] for r in rows if r.get("url")))
|
||
result = {"status": status, "reason": reason, "accepted_for_delivery": status in ("ok", "no_answer", "ambiguous"),
|
||
"answer": text, "source_urls": urls, "claims": [], "index_version": None,
|
||
"complete": complete, "truncated": truncated, "truncation_reasons": list(reasons),
|
||
"delivery_check": {"layer": {"ok": "delivery", "ambiguous": "planning"}.get(status, "retrieval")
|
||
if reason not in ("invalid_context", "expired_context") else "context",
|
||
"code": ("partial" if truncated else "complete") if status == "ok"
|
||
else "evidence_not_found" if status == "no_answer"
|
||
else "clarification_required" if status == "ambiguous" else "unavailable",
|
||
"request_fingerprint": fingerprint(question, ENGINE)}}
|
||
if count is not None:
|
||
result["count"] = count
|
||
return result
|
||
|
||
|
||
def answer(question: str, *, limit: int = 5, complete=None, context_token: str | None = None) -> dict:
|
||
key = (question.strip(), limit, context_token or "")
|
||
with _cache_lock:
|
||
hit = _cache.get(key)
|
||
if hit and time.monotonic() - hit[0] < CACHE_SECONDS and complete is None:
|
||
return hit[1]
|
||
result = _answer(question.strip(), complete, context_token)
|
||
if result["status"] == "ok" and complete is None:
|
||
with _cache_lock:
|
||
_cache[key] = (time.monotonic(), result)
|
||
while len(_cache) > 256:
|
||
_cache.popitem(last=False)
|
||
return result
|
||
|
||
|
||
def other_roles(rows: list[dict], query: CrzpQuery) -> str:
|
||
""""Hládek viedol 0 bakalárok v 2023" may still be his opponent role in such works."""
|
||
if not query.people or query.people[0].role not in ROLE_FIELD:
|
||
return ""
|
||
person = query.people[0]
|
||
relaxed = query.model_copy(deep=True)
|
||
relaxed.people[0].role = "any"
|
||
roles = Counter(role for r in rows if keep(r, relaxed) for role, field in ROLE_FIELD.items()
|
||
if role != person.role and any(name_matches(person.name, n) for n in r.get(field, [])))
|
||
if not roles:
|
||
return ""
|
||
parts = ", ".join(f"{ROLE_LABEL[role]} pri {c} {'práci' if c == 1 else 'prácach'}" for role, c in roles.most_common())
|
||
return f"\n\nS rovnakými ostatnými podmienkami je {person.name} v CRZP v inej role: {parts}."
|
||
|
||
|
||
def _without_suffix(query: CrzpQuery, texts: list[str]) -> list[str]:
|
||
"""Searched texts without the school or workplace appended to each of them."""
|
||
return list(dict.fromkeys(strip_suffix(query, t) for t in texts))
|
||
|
||
|
||
def _answer(question: str, complete=None, context_token: str | None = None) -> dict:
|
||
previous = None
|
||
if context_token:
|
||
try:
|
||
previous = context.restore(context_token)
|
||
except context.ContextError as error:
|
||
return _result(question, "error", "Kontext predchádzajúcej otázky už neplatí; polož otázku celú "
|
||
"(meno, téma, rok).", reason=str(error))
|
||
if complete is None:
|
||
fetch.prepare() # CRZP home page loads while the model reads the question
|
||
query = understand(question, complete=complete)
|
||
if query.intent == "refused":
|
||
return _result(question, "no_answer", "Pokyny na zmenu pravidiel nevykonávam a celý obsah CRZP, prihlasovacie "
|
||
"údaje ani systémové nastavenia nevypisujem. Opýtaj sa na konkrétnu prácu, osobu, školu, tému "
|
||
"alebo rok.", reason="request_out_of_scope")
|
||
if previous:
|
||
query, clarification = context.follow_up(question, query, previous)
|
||
if clarification:
|
||
result = _result(question, "ambiguous", clarification, reason="clarification_required")
|
||
result["conversation_context"] = {"token": context_token, "work_options": [], "meeting_options": []}
|
||
return result
|
||
result = _answer_query(question, query, complete)
|
||
if result["status"] in ("ok", "no_answer", "ambiguous") and result.get("reason") != "request_out_of_scope":
|
||
conversation = context.issue(question, result.pop("_asked", query), result.pop("_shown", []),
|
||
result.pop("_options", None))
|
||
if conversation:
|
||
result["conversation_context"] = conversation
|
||
result.pop("_shown", None), result.pop("_asked", None), result.pop("_options", None)
|
||
return result
|
||
|
||
|
||
def _answer_query(question: str, query: CrzpQuery, complete=None) -> dict:
|
||
asked = query.model_copy(deep=True) # before find() adapts names and roles: the follow-up repeats this
|
||
if not query.search_text():
|
||
return _result(question, "no_answer", "Na vyhľadanie v CRZP uveď meno študenta, vedúceho alebo oponenta, "
|
||
"názov práce alebo tému.", reason="missing_search_terms")
|
||
try:
|
||
found, rows = find(query, links=query.intent != "count")
|
||
if found is None:
|
||
return _result(question, "no_answer", "Otázke nezodpovedá žiadne vyhľadávanie v CRZP.",
|
||
reason="missing_search_terms")
|
||
searched = "„" + "“, „".join(found["texts"]) + "“"
|
||
if not rows:
|
||
tried = "„" + "“, „".join(_without_suffix(query, query.searched)) + "“"
|
||
result = _result(question, "no_answer",
|
||
f"V CRZP som nenašiel prácu, ktorá by zodpovedala otázke ({describe(query)}). "
|
||
f"Hľadal som {tried}; žiadny nájdený záznam nespĺňa všetky podmienky. "
|
||
"CRZP hľadá presné tvary slov, preto to nemusí znamenať, že taká práca neexistuje."
|
||
+ other_roles(found["rows"], query),
|
||
reason="no_matching_record", complete=found["complete"])
|
||
result["_shown"], result["_asked"] = [], asked
|
||
return result
|
||
asked_who = clarification(rows, query)
|
||
if asked_who:
|
||
result = _result(question, "ambiguous", asked_who[0], reason="clarification_required")
|
||
result["_shown"], result["_asked"], result["_options"] = [], asked, asked_who[1]
|
||
return result
|
||
handler = {"list": _list, "count": _count, "keywords": _keywords, "relation": _relation,
|
||
"similar": _similar, "schools": _schools}.get(query.intent)
|
||
if query.intent == "about":
|
||
body, used = _about(question, query, found, rows, complete)
|
||
else:
|
||
body, used = handler(question, query, found, rows)
|
||
except CrzpUnavailable:
|
||
return _result(question, "error", "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.",
|
||
reason="crzp_unavailable")
|
||
if getattr(query, "relaxed_author", False):
|
||
body = (f"{query.people[0].name} nie je v CRZP autorom žiadnej z nájdených prác; uvedené sú práce, pri ktorých "
|
||
"je školiteľom alebo oponentom.\n\n" + body)
|
||
if query.intent in ("about", "keywords", "list"):
|
||
body += namesakes(used, query)
|
||
if query.unverified_only:
|
||
body = (f"Výraz „{query.topic_key()}“ sa v CRZP nenašiel; uvedené práce sú nájdené pod navrhnutým "
|
||
f"príbuzným výrazom „{'“, „'.join(query.unverified_only)}“ (návrh, nie overené synonymum).\n\n" + body)
|
||
terms = _without_suffix(query, found["texts"])
|
||
if len(terms) > 1 and query.topics:
|
||
body += "\n\nHľadané výrazy v CRZP: „" + "“, „".join(terms) + "“ (vrátane príbuzných výrazov témy)."
|
||
truncated = not found["complete"]
|
||
reasons = ["crzp_result_limit"] if truncated else []
|
||
for text, total, read in found["incomplete"]:
|
||
body += (f"\n\nPoznámka: hľadanie „{text}“ vrátilo v CRZP {total} záznamov; prehľadaných bolo prvých {read}.")
|
||
body += "\n\nZdroj: Centrálny register záverečných prác (CRZP), metadáta záznamov."
|
||
result = _result(question, "ok", body, rows=used, complete=not truncated, truncated=truncated, reasons=reasons,
|
||
count=len(rows) if query.intent == "count" else None)
|
||
result["_shown"], result["_asked"] = used, asked
|
||
return result
|