1041 lines
56 KiB
Python
1041 lines
56 KiB
Python
"""ZP Wiki and CRZP in one answer for /rag and /graphrag.
|
||
|
||
Every question is answered from ZP Wiki (the normal RAG / GraphRAG path) and,
|
||
at the same time, from CRZP when CRZP can add something (see crzp_plan):
|
||
- a person ZP Wiki knows: that person's works in CRZP at Technická univerzita
|
||
v Košiciach (supervisor, opponent, abstract, keywords the wiki may lack);
|
||
- a topic or "similar works": CRZP works on the same topic, with their schools;
|
||
- a question naming CRZP, another school, opponents or defended works of a
|
||
department: CRZP is the main source.
|
||
Meetings, tasks, progress and results stay ZP Wiki only.
|
||
|
||
The answer is one text: a list of people is one numbered list (ZP Wiki people
|
||
first, then CRZP works, each marked with its source), the first PAGE items are
|
||
shown and the rest is offered ("Chceš vidieť ďalších?"); all sources are in one
|
||
line at the end. A signed token carries the wiki's own token, the shown people
|
||
(for "a z akých škôl sú?", "a čo robil prvý z nich?") and the id of the not
|
||
yet shown items (tools/answers/pages.py).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import hashlib
|
||
import hmac
|
||
import json
|
||
import os
|
||
import re
|
||
import threading
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout
|
||
from functools import lru_cache
|
||
|
||
from tools.answers.context_policy import normalized
|
||
from tools.answers.delivery_contract import fingerprint
|
||
|
||
CRZP_SECONDS = float(os.getenv("CRZP_COMBINED_SECONDS", "45"))
|
||
LIFETIME = 24 * 3600
|
||
PREFIX = "zc1."
|
||
TUKE = "Technická univerzita v Košiciach"
|
||
WIKI_SCHOOL = "Technická univerzita v Košiciach (KEMT FEI)"
|
||
# What only the wiki records: CRZP is not asked.
|
||
WIKI_ONLY = re.compile(r"\b(?:stretnut\w*|pokrok\w*|progres\w*|uloh\w*|zadan\w*|plan\w*|termin\w*|tyzd\w*|"
|
||
r"repozitar\w*|git\w*|stav\w*|poznam\w*|konzult\w*|zaciat\w*|studi\w*|obhaj\w*|ukonc\w*|"
|
||
r"dokonc\w*|uspes\w*|statnic\w*|hodnot\w*|znamk\w*|problem\w*|vtedy|posledn\w*)\b")
|
||
EXPLICIT = re.compile(r"\b(?:crzp|registr\w*|register|celoslovensk\w*|na slovensku|inych skol|inej skole|"
|
||
r"vsetkych skol|ostatnych skol|oponent\w*|oponoval\w*)\b")
|
||
METHODS = re.compile(r"\b(?:technolog\w*|nastroj\w*|metod\w*|pouziv\w*|pouzil\w*|vyuziv\w*|vyuzil\w*|framework\w*)\b")
|
||
ABOUT = re.compile(r"\b(?:o com|zhrn\w*|abstrakt\w*|ciel\w*)\b") # not "čomu sa venuje": the current work
|
||
# A question about a thesis (not "na čom pracuje", which CRZP's submitted works cannot answer).
|
||
THESIS = re.compile(r"\b(?:prac(?:a|e|u|i|ou|ach|ami)?|bakalar\w*|diplom\w*|dizerta\w*|habilita\w*|tem(?:a|u|y|ou)|"
|
||
r"oponent\w*|vedu\w*|skolitel\w*|abstrakt\w*|klucov\w*|tagy?)\b")
|
||
WIKI_CLARIFICATION_YIELDS = {"missing_subject", "unsupported_content_question", "unsupported_constraint"}
|
||
# "Koľko prác obhájili na KEMT v roku 2024?": defended works of a school or department are CRZP's
|
||
# published works, not a wiki question about one student's defence.
|
||
DEFENDED = re.compile(r"\b(?:obhaj\w*|obhajen\w*|obhajil\w*|uspesn\w*)\b")
|
||
TOPIC_MIN_SCORE = float(os.getenv("CRZP_TOPIC_MIN_SCORE", "0"))
|
||
PAGE = int(os.getenv("ANSWER_PAGE_SIZE", "6"))
|
||
WIKI_FIRST = 4 # a mixed list shows at most this many ZP Wiki people before the CRZP works
|
||
REFUSED = {"request_out_of_scope", "irrelevant", "unsupported_request", "out_of_scope"}
|
||
RELATED = re.compile(r"\b(?:suvis\w*|pribuzn\w*)\b")
|
||
SIMILAR = re.compile(r"\b(?:podobn\w*|suvisiac\w*|pribuzn\w*|spolupra\w*|rovnak\w* tem\w*)\b")
|
||
TOPIC_LIST = re.compile(r"^\W*(?:kto|ktor\w*|ak\w*)\b")
|
||
SCHOOLS = re.compile(r"\b(?:z|zo|na) (?:akych|akej|ktorych|ktorej) (?:skol|univerzit|fakult)\w*|\bkde (?:studoval\w*|studuj\w*)"
|
||
r"|\b(?:ktor|ak)\w* (?:skol|univerzit)\w*")
|
||
CHOICE_WIKI = re.compile(r"\b(?:wiki|diplom\w*|dp)\b")
|
||
CHOICE_CRZP = re.compile(r"\b(?:crzp|bakalar\w*|bp)\b")
|
||
|
||
|
||
_shared = {"pool": None}
|
||
_shared_lock = threading.Lock()
|
||
|
||
|
||
def _pool() -> ThreadPoolExecutor:
|
||
"""One pool for all questions (no new threads per request)."""
|
||
with _shared_lock:
|
||
if _shared["pool"] is None:
|
||
_shared["pool"] = ThreadPoolExecutor(max_workers=int(os.getenv("ANSWER_COMBINED_WORKERS", "6")),
|
||
thread_name_prefix="answer")
|
||
return _shared["pool"]
|
||
|
||
|
||
def enabled() -> bool:
|
||
return os.getenv("ANSWER_WITH_CRZP", "1") != "0"
|
||
|
||
|
||
# ------------------------------------------------------------------ token
|
||
|
||
def _key() -> bytes | None:
|
||
secret = os.getenv("SEARCH_API_KEY", "")
|
||
return hashlib.sha256(("zp-combined-v1\0" + secret).encode()).digest() if secret else None
|
||
|
||
|
||
def _b64(value: bytes) -> str:
|
||
return base64.urlsafe_b64encode(value).decode().rstrip("=")
|
||
|
||
|
||
def seal(payload: dict) -> str | None:
|
||
key = _key()
|
||
if key is None:
|
||
return None
|
||
body = _b64(json.dumps({**payload, "expires": int(time.time()) + LIFETIME}, ensure_ascii=False,
|
||
separators=(",", ":")).encode())
|
||
token = PREFIX + body + "." + _b64(hmac.digest(key, body.encode(), "sha256"))
|
||
return token if len(token) <= 16000 else None
|
||
|
||
|
||
def unseal(token: str | None) -> dict | None:
|
||
"""The combined context, or None for a token that is not ours (a plain wiki token)."""
|
||
if not isinstance(token, str) or not token.startswith(PREFIX) or _key() is None:
|
||
return None
|
||
try:
|
||
body, signature = token[len(PREFIX):].split(".")
|
||
expected = hmac.digest(_key(), body.encode(), "sha256")
|
||
if not hmac.compare_digest(base64.b64decode(signature + "=" * (-len(signature) % 4), altchars=b"-_"), expected):
|
||
raise ValueError
|
||
value = json.loads(base64.b64decode(body + "=" * (-len(body) % 4), altchars=b"-_"))
|
||
if value.get("expires", 0) < time.time():
|
||
return {"error": "expired_context"}
|
||
return value
|
||
except (ValueError, TypeError, UnicodeError):
|
||
return {"error": "invalid_context"}
|
||
|
||
|
||
# ------------------------------------------------------------------ wiki facts
|
||
|
||
_index_cache: dict = {}
|
||
_index_lock = threading.Lock()
|
||
|
||
|
||
def _index():
|
||
"""The published knowledge index, reloaded only when its version changes."""
|
||
from tools.knowledge.store import load_index
|
||
from tools.rag.common import DB_FILE
|
||
from tools.sync.index_state import read_version
|
||
version = read_version(DB_FILE.parent)
|
||
key = (version or {}).get("version")
|
||
with _index_lock:
|
||
if _index_cache.get("key") != key:
|
||
_index_cache.update(key=key, index=load_index(DB_FILE.parent, version))
|
||
return _index_cache["index"]
|
||
|
||
|
||
def wiki_units(person: str) -> list[dict]:
|
||
try:
|
||
return [u for u in (_index() or {}).get("units", []) if u.get("person") == person]
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
def wiki_works(person: str) -> dict[str, dict]:
|
||
"""work label -> {"type", "topic"} of the person's ZP Wiki works."""
|
||
works = {}
|
||
for unit in wiki_units(person):
|
||
label = unit.get("work_label")
|
||
if not label:
|
||
continue
|
||
work = works.setdefault(label, {"type": unit.get("work_type"), "topic": None})
|
||
if unit.get("kind") == "topic" and not work["topic"] and not unit["text"].strip().endswith(":"):
|
||
work["topic"] = re.sub(r"^\s*(?:Téma|Názov)\s*:\s*", "", unit["text"]).strip()
|
||
return works
|
||
|
||
|
||
FRAMEWORKS = {"python", "fastapi", "flask", "django", "streamlit", "react", "docker", "kubernetes", "git", "github",
|
||
"gitlab", "pandas", "numpy", "jupyter", "colab", "linux", "javascript", "typescript", "java", "html", "css",
|
||
"llm", "ai", "nlp", "veľký jazykový model", "umelá inteligencia", "zpwiki", "openwebui", "skills", "skill"}
|
||
|
||
|
||
def wiki_terms(person: str) -> list[str]:
|
||
"""The person's most specific topics and tools (reviewed topic groups first, frameworks never):
|
||
the words to look for in CRZP to find similar works."""
|
||
from tools.answers.content_selection import named_tools
|
||
from tools.retrieval.synonyms import base_name, group_of
|
||
counts: dict[str, float] = {}
|
||
for unit in wiki_units(person):
|
||
if unit.get("phase") not in {"result", "progress", "assignment"} and unit.get("kind") != "topic":
|
||
continue
|
||
for concept in unit.get("concepts") or []:
|
||
if not isinstance(concept, str):
|
||
continue
|
||
if len(group_of(concept)) > 1:
|
||
name, weight = base_name(concept), 2.0
|
||
elif named_tools(concept) and len(concept.split()) <= 2 and not re.search(r"[./]", concept):
|
||
name, weight = concept, 1.0
|
||
else:
|
||
continue
|
||
if normalized(name) not in FRAMEWORKS:
|
||
counts[name] = counts.get(name, 0) + weight
|
||
return sorted(counts, key=lambda n: (-counts[n], n))[:4]
|
||
|
||
|
||
def wiki_profile(person: str) -> str:
|
||
"""Topic titles and terms of the person's works: the text similar CRZP works are ranked against."""
|
||
works = wiki_works(person)
|
||
return ". ".join([*(w["topic"] for w in works.values() if w.get("topic")), ", ".join(wiki_terms(person))])
|
||
|
||
|
||
# ------------------------------------------------------------------ CRZP part
|
||
|
||
def crzp_plan(question: str, person: str | None, complete=None, student: bool = True):
|
||
"""(mode, CrzpQuery) or None when CRZP adds nothing to this question."""
|
||
from tools.crzp.answering.understand import Person, understand
|
||
from tools.crzp.answering.units import unit_in_question
|
||
low = normalized(question)
|
||
explicit = bool(EXPLICIT.search(low))
|
||
defended_scope = bool(DEFENDED.search(low) and not person and (
|
||
unit_in_question(question) or re.search(r"\b(?:univerzit\w*|fakult\w*|katedr\w*|skol\w*|tuke|stu|kemt|fei)\b", low)))
|
||
if WIKI_ONLY.search(low) and not explicit and not defended_scope:
|
||
return None
|
||
# A person ZP Wiki knows needs no model reading of the question (the person and school are
|
||
# fixed below; rules give the intent): one model call less, the wiki part has the slots.
|
||
query = understand(question, complete=complete, use_model=not person or explicit)
|
||
if query.intent == "refused":
|
||
return None
|
||
other_school = query.school not in (None, "tuke") or explicit
|
||
if person and other_school and not query.from_model:
|
||
query = understand(question, complete=complete)
|
||
if person and not other_school:
|
||
if SIMILAR.search(low) and student:
|
||
return "similar", query
|
||
role = query.people[0].role if query.people else "any"
|
||
if student:
|
||
role = "author" if role == "author" else "any" # "Kto vedie Jána Ptáka?": his own works
|
||
elif role not in ("supervisor", "opponent"):
|
||
role = "supervisor" # a supervisor's works are those he or she led
|
||
query.people = [Person(name=person, role=role, typed=person)]
|
||
query.school, query.unit, query.from_model, query.title = "tuke", None, True, None
|
||
query.topics, query.alternatives = [], []
|
||
return "person", query
|
||
if query.people and not person and query.search_text() and THESIS.search(low):
|
||
return "explicit", query # a person ZP Wiki does not know: CRZP knows the theses of all schools
|
||
if query.topics and not query.people and RELATED.search(low) and not re.search(r"\b(?:tag\w*|klucov\w*)\b", low):
|
||
query.intent = "keywords" # "Ktoré témy súvisia s X?": topics of works about X, and who did them
|
||
return "related", query
|
||
if query.topics and not query.people and RELATED.search(low):
|
||
query.intent = "keywords" # "Ktoré tagy súvisia s X?": the CRZP tags themselves
|
||
return "explicit", query
|
||
if query.topics and not query.people and query.intent in ("relation", "count", "keywords", "schools"):
|
||
return "explicit", query # "Kto viedol najviac prác o X?", "Koľko prác o X…": computed over CRZP
|
||
if (other_school or ((query.school or query.unit) and not query.people)) and query.search_text():
|
||
return "explicit", query # another school, CRZP itself, opponents, or a school/department as a whole
|
||
if query.topics and not query.people and (TOPIC_LIST.search(low) or SIMILAR.search(low)):
|
||
return "topic", query
|
||
return None
|
||
|
||
|
||
def clean_crzp(text: str) -> str:
|
||
"""CRZP answer text inside a combined answer: its own source lines go to the common "Zdroje CRZP" line."""
|
||
lines = [l for l in text.split("\n") if not re.match(r"^\s*(?:-\s*)?Zdroj:\s", l)]
|
||
return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()
|
||
|
||
|
||
def explicit_part(question: str, result: dict) -> dict | None:
|
||
"""A whole CRZP answer (CRZP is the main source), with its conversation token for follow-ups."""
|
||
from tools.crzp.answering import answer as crzp, context as crzp_context
|
||
if result.get("status") not in ("ok", "ambiguous"):
|
||
return None
|
||
rows = result.pop("_shown", None) or []
|
||
asked = result.pop("_asked", None)
|
||
options = result.pop("_options", None)
|
||
token = None
|
||
if asked is not None:
|
||
issued = crzp_context.issue(question, asked, rows, options)
|
||
token = issued["token"] if issued else None
|
||
entities = result.pop("_entities", None) or []
|
||
return {"mode": "explicit", "status": result["status"], "text": clean_crzp(result["answer"]), "rows": rows,
|
||
"urls": result.get("source_urls", []), "crzp_token": token, "entities": entities}
|
||
|
||
|
||
def _kind(row) -> str:
|
||
from tools.crzp.answering.answer import _kind as kind
|
||
return kind(row)
|
||
|
||
|
||
def _names(values) -> str:
|
||
from tools.crzp.source.parse import display_name
|
||
return ", ".join(display_name(v) for v in values) or "neuvedený"
|
||
|
||
|
||
def _work(row) -> str:
|
||
return f"{_kind(row)} {row.get('year') or ''}".strip() + f" „{row['title']}“"
|
||
|
||
|
||
def crzp_methods(person: str, row: dict, detail: dict | None) -> str:
|
||
""""Ján Pták vo svojej bakalárskej práci „…“ (2025) využíva: ESP32, MQTT, InfluxDB, Home Assistant." —
|
||
the work's keywords and the tools its abstract names."""
|
||
from tools.answers.content_selection import named_tools
|
||
record = detail or row
|
||
abstract = record.get("abstract") or ""
|
||
found = []
|
||
for keyword in record.get("keywords", []):
|
||
spelled = re.search(r"(?i)(?<!\w)" + re.escape(keyword) + r"(?!\w)", abstract)
|
||
name = spelled.group(0) if spelled else keyword
|
||
if named_tools(name) or re.search(r"[A-Z]", name[1:] if spelled else name): # "Home Assistant", not "časová značka"
|
||
found.append(name)
|
||
found += [t.rstrip(".,;:") for t in named_tools(abstract)]
|
||
tools = []
|
||
for name in found:
|
||
key = normalized(name)
|
||
if key and len(key) > 1 and not any(f" {key} " in f" {normalized(t)} " for t in tools):
|
||
tools = [t for t in tools if f" {normalized(t)} " not in f" {key} "]
|
||
tools.append(name)
|
||
where = {"bachelor_thesis": "vo svojej bakalárskej práci", "diploma_thesis": "vo svojej diplomovej práci",
|
||
"dissertation": "vo svojej dizertačnej práci"}.get(row.get("work_type"), "vo svojej práci")
|
||
return f"{person} {where} „{row['title']}“ ({row.get('year')}) využíva: {', '.join(tools[:12])}."
|
||
|
||
|
||
def remember(rows: list[dict]) -> None:
|
||
"""Project the used CRZP works into Neo4j, linked to ZP Wiki people and topics (in the background)."""
|
||
from tools.crzp.storage import graph
|
||
rows = [r for r in rows if r.get("sid")]
|
||
if rows and os.getenv("CRZP_GRAPH_LINKS", "1") != "0":
|
||
_pool().submit(graph.project, rows)
|
||
|
||
|
||
def crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None:
|
||
"""The CRZP contribution: {"mode", "text", "rows", "people", "works"} or None."""
|
||
part = _crzp_part(question, person, complete=complete, student=student)
|
||
if part:
|
||
remember(part.get("rows") or [])
|
||
return part
|
||
|
||
|
||
def _crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None:
|
||
from tools.crzp.answering import answer as crzp
|
||
from tools.crzp.source.client import CrzpUnavailable
|
||
try:
|
||
planned = crzp_plan(question, person, complete, student)
|
||
if planned is None:
|
||
return None
|
||
mode, query = planned
|
||
low = normalized(question)
|
||
if mode == "similar":
|
||
return crzp_similar(person, complete=complete)
|
||
if mode == "related":
|
||
return related_part(query)
|
||
if mode == "explicit":
|
||
return explicit_part(question, crzp._answer_query(question, query, complete))
|
||
found, rows = crzp.find(query)
|
||
if not rows:
|
||
return None
|
||
if mode == "topic":
|
||
rows = relevant(question, query, rows)[:12]
|
||
if not rows:
|
||
return None # nothing in CRZP is really about the topic: no filler works
|
||
return {"mode": mode, "items": [crzp_item(r) for r in rows], "rows": rows, "text": "",
|
||
"urls": [r["url"] for r in rows if r.get("url")]}
|
||
rows = crzp.ranked(question, query, rows)
|
||
# person: that person's works at TUKE
|
||
rows = [r for r in rows if crzp.university(r) == TUKE]
|
||
if not rows:
|
||
return None
|
||
if not student: # a supervisor: the works he or she led, newest first
|
||
rows = sorted(rows, key=lambda r: -(r.get("year") or 0))
|
||
lines = [f"- {_names(r.get('authors', []))} — „{r['title']}“ ({_kind(r)}, {r.get('year')})" for r in rows[:8]]
|
||
more = f"\n\nSpolu {len(rows)} prác v CRZP na TUKE; zobrazených je 8 najnovších." if len(rows) > 8 else ""
|
||
return {"mode": mode, "text": f"Práce, ktoré {person} viedol alebo oponoval, v CRZP (TUKE):\n\n"
|
||
+ "\n".join(lines) + more, "rows": rows[:8], "urls": [r["url"] for r in rows[:8] if r.get("url")]}
|
||
lines, details = [], {}
|
||
for row in rows[:3]:
|
||
if ABOUT.search(low) or METHODS.search(low):
|
||
details[row.get("sid")] = crzp.fetch.detail(row["sid"]) if row.get("sid") else None
|
||
detail = details.get(row.get("sid"))
|
||
if METHODS.search(low):
|
||
lines.append(crzp_methods(person, row, detail))
|
||
elif ABOUT.search(low) and detail:
|
||
label, text = crzp.summary(detail, complete)
|
||
lines.append(f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{text}")
|
||
else:
|
||
lines.append(f"- {_work(row)} — vedúci {_names(row.get('supervisors', []))}; "
|
||
f"oponent {_names(row.get('opponents', []))}")
|
||
return {"mode": mode, "text": "\n\n".join(lines), "rows": rows[:3],
|
||
"urls": [r["url"] for r in rows[:3] if r.get("url")],
|
||
"works": [{"source": "crzp", "label": _work(r), "type": r.get("work_type"), "key": crzp.work_key(r)}
|
||
for r in rows[:3]]}
|
||
except CrzpUnavailable:
|
||
return None
|
||
|
||
|
||
def crzp_similar(person: str, *, complete=None) -> dict | None:
|
||
"""CRZP works sharing the person's specific tools/topics (searched in CRZP by them)."""
|
||
from tools.crzp import fetch
|
||
from tools.crzp.answering import answer as crzp
|
||
from tools.crzp.source.parse import fold
|
||
terms = wiki_terms(person)
|
||
if not terms:
|
||
return None
|
||
scored: dict = {}
|
||
for term in terms:
|
||
for row in fetch.search(term)["rows"]:
|
||
if any(fold(person.split()[-1]) in fold(a) for a in row.get("authors", [])):
|
||
continue
|
||
text = fold(row["title"] + " " + " ".join(row.get("keywords", [])))
|
||
shared = [t for t in terms if fold(t) in text]
|
||
if shared:
|
||
key = crzp.work_key(row)
|
||
scored[key] = (row, sorted(set(shared) | set(scored.get(key, (None, []))[1])))
|
||
if not scored:
|
||
return None
|
||
candidates = list(scored.values())
|
||
from tools.retrieval import rerank
|
||
profile = wiki_profile(person)
|
||
order = rerank.scores(profile, [r["title"] + ". " + ", ".join(r.get("keywords", [])) for r, _ in candidates]) \
|
||
or [0] * len(candidates)
|
||
# Closest by content first; more shared topics/tools break ties.
|
||
best = [c for _, c in sorted(zip(order, candidates), key=lambda i: (-i[0], -len(i[1][1])))][:12]
|
||
return {"mode": "similar", "items": [crzp_item(r, shared) for r, shared in best], "text": "",
|
||
"rows": [r for r, _ in best], "urls": [r["url"] for r, _ in best if r.get("url")]}
|
||
|
||
|
||
_tag_cache: dict = {}
|
||
_tag_lock = threading.Lock()
|
||
|
||
|
||
def tag_frequencies() -> tuple[dict, int]:
|
||
"""In how many cached CRZP works each tag (its reviewed base name) occurs, and of how many works:
|
||
a tag most works carry ("umelá inteligencia", "python") names no specific related topic."""
|
||
from collections import Counter
|
||
from tools.crzp.fetch import store
|
||
stamp = int(time.time() // 1800) # recounted at most every half hour
|
||
with _tag_lock:
|
||
if _tag_cache.get("stamp") == stamp:
|
||
return _tag_cache["df"], _tag_cache["total"]
|
||
from tools.retrieval.synonyms import base_name
|
||
counts, seen = Counter(), set()
|
||
for row in store().cached_rows():
|
||
key = (row.get("title"), row.get("year"))
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
counts.update({normalized(base_name(k.strip())) for k in row.get("keywords", []) if k.strip()})
|
||
with _tag_lock:
|
||
_tag_cache.update(stamp=stamp, df=dict(counts), total=max(1, len(seen)))
|
||
return _tag_cache["df"], _tag_cache["total"]
|
||
|
||
|
||
def wiki_people_for(topic: str) -> list[str]:
|
||
"""ZP Wiki students whose own records (topic, assignment, progress, results) name the topic."""
|
||
from tools.retrieval.synonyms import variants_pattern
|
||
pattern = variants_pattern(topic)
|
||
people = {}
|
||
for unit in (_index() or {}).get("units", []):
|
||
if unit.get("kind") in {"reference", "meeting", "supervisor", "consultant", "lifecycle"}:
|
||
continue
|
||
if pattern.search(normalized(unit.get("text", ""))) or any(
|
||
pattern.fullmatch(normalized(c)) for c in unit.get("concepts", []) if isinstance(c, str)):
|
||
people.setdefault(unit["person"], unit.get("source_url"))
|
||
return list(people)
|
||
|
||
|
||
def related_part(query) -> dict | None:
|
||
"""Topics related to the asked one: the CRZP keywords of works about it, weighted by how specific
|
||
they are, each with the ZP Wiki students and CRZP works that have it."""
|
||
import math
|
||
from tools.crzp.answering import answer as crzp
|
||
from tools.retrieval.synonyms import base_name, group_of, variants_pattern
|
||
found, rows = crzp.find(query)
|
||
if not rows:
|
||
return None
|
||
asked = [t for t in [*query.topics, *query.phrases] if t]
|
||
patterns = [variants_pattern(t) for t in asked]
|
||
asked_words = {w for t in asked for x in group_of(t) for w in normalized(x).split()}
|
||
generic = {normalized(g) for g in crzp.GENERIC_TAGS} | {"umela inteligencia", "strojove ucenie", "python"}
|
||
df, total = tag_frequencies()
|
||
stats: dict = {}
|
||
for row in rows:
|
||
for keyword in {base_name(k.strip()) for k in row.get("keywords", []) if k.strip()}:
|
||
key = normalized(keyword)
|
||
if (len(key) < 3 or key in generic or set(key.split()) <= asked_words
|
||
or any(p.fullmatch(key) for p in patterns)):
|
||
continue # the asked topic itself, a synonym, a part of it or a generic word
|
||
entry = stats.setdefault(key, {"label": keyword, "rows": []})
|
||
if keyword[:1].isupper() and not entry["label"][:1].isupper():
|
||
entry["label"] = keyword
|
||
entry["rows"].append(row)
|
||
ranked = sorted((e for e in stats.values() if len(e["rows"]) >= 2),
|
||
key=lambda e: -len(e["rows"]) * math.log(total / (1 + df.get(normalized(e["label"]), 0))))[:18]
|
||
if not ranked:
|
||
return None
|
||
items = []
|
||
for entry in ranked:
|
||
label = entry["label"][:1].upper() + entry["label"][1:]
|
||
example = max(entry["rows"], key=lambda r: r.get("year") or 0)
|
||
authors = example.get("authors") or []
|
||
from tools.crzp.source.parse import display_name
|
||
wiki = wiki_people_for(entry["label"])
|
||
text = (f"CRZP: {crzp.plural(len(entry['rows']))}, napr. {display_name(authors[0]) if authors else 'autor neuvedený'}"
|
||
f" — {crzp.university(example)}: „{example['title']}“ ({example.get('year') or 'rok neuvedený'})")
|
||
if wiki:
|
||
text = "ZP Wiki: " + ", ".join(wiki[:4]) + (f" a ďalší ({len(wiki) - 4})" if len(wiki) > 4 else "") + "; " + text
|
||
items.append({"name": label, "source": "topic", "text": text,
|
||
"urls": [example["url"]] if example.get("url") else []})
|
||
topic = base_name(asked[0]) if asked else ""
|
||
return {"mode": "related", "items": items, "topic": topic, "rows": [e["rows"][0] for e in ranked], "text": "",
|
||
"urls": [i["urls"][0] for i in items if i["urls"]]}
|
||
|
||
|
||
def crzp_item(row: dict, shared: list[str] | None = None) -> dict:
|
||
"""One CRZP work as an item of a combined list."""
|
||
from tools.crzp.answering import answer as crzp
|
||
from tools.crzp.source.parse import display_name
|
||
authors = row.get("authors") or []
|
||
name = display_name(authors[0]) if authors else "autor neuvedený"
|
||
school = crzp.university(row)
|
||
line = f"{school}: „{row['title']}“ ({_kind(row)}, {row.get('year') or 'rok neuvedený'})"
|
||
if shared:
|
||
line += "; spoločné: " + ", ".join(shared)
|
||
return {"name": name, "school": school, "source": "crzp", "text": line, "urls": [row["url"]] if row.get("url") else [],
|
||
"sid": row.get("sid"), "title": row["title"]}
|
||
|
||
|
||
def relevant(question: str, query, rows: list[dict]) -> list[dict]:
|
||
"""CRZP works really about the asked topic, most relevant first: the topic (or a reviewed
|
||
synonym, in any case form) in the title or the CRZP keywords, or a cross-encoder score
|
||
above CRZP_TOPIC_MIN_SCORE. A work found only through its abstract ("Fairy Tale
|
||
Intertexts" for "anotácie") is no answer to a topic question."""
|
||
from tools.retrieval import rerank
|
||
from tools.retrieval.synonyms import variants_pattern
|
||
if not rows:
|
||
return []
|
||
terms = [t for t in [*query.topics, *query.phrases] if t]
|
||
patterns = [variants_pattern(t) for t in terms]
|
||
texts = [r["title"] + ". " + ", ".join(r.get("keywords", [])) for r in rows]
|
||
asked = " ".join([question, *[t for t in [*query.topics, *query.alternatives] if normalized(t) not in normalized(question)]])
|
||
scores = rerank.scores(asked, texts)
|
||
if scores is not None: # the cross-encoder scores at most MAX_PAIRS texts; the rest only by name
|
||
scores = list(scores) + [float("-inf")] * (len(texts) - len(scores))
|
||
kept = []
|
||
for i, (row, text) in enumerate(zip(rows, texts)):
|
||
named = any(p.search(normalized(text)) for p in patterns)
|
||
score = scores[i] if scores is not None else (1.0 if named else -1.0)
|
||
if named or score >= TOPIC_MIN_SCORE:
|
||
kept.append((score + (1.0 if named else 0.0), i, row))
|
||
return [row for _, _, row in sorted(kept, key=lambda k: (-k[0], k[1]))]
|
||
|
||
|
||
# ------------------------------------------------------------------ items of a list answer
|
||
|
||
HEADING = re.compile(r"^\*\*([^*\n]+?)\*\*\s*$")
|
||
PERSON_NAME = re.compile(r"[A-ZÁ-Ž][\w'’.-]+(?:\s+[A-ZÁ-Ž][\w'’.-]+){1,3}")
|
||
# Blocks after the people of a wiki list (they belong to the whole list).
|
||
TAIL = re.compile(r"^(?:Téma je len v zadaní|Len štúdium alebo poznámky|Tému majú len|Podklady k porovnávanej osobe)")
|
||
SOURCE_LINE = re.compile(r"^Zdroj(?:e)?(?: CRZP)?:\s*(.+)$", re.M)
|
||
|
||
|
||
def split_sources(text: str) -> tuple[str, list[str]]:
|
||
"""The answer without its source lines, and their links."""
|
||
urls = []
|
||
for match in SOURCE_LINE.finditer(text or ""):
|
||
urls += re.findall(r"https?://[^\s,)\]]+", match.group(1))
|
||
body = SOURCE_LINE.sub("", text or "")
|
||
return re.sub(r"\n{3,}", "\n\n", body).strip(), list(dict.fromkeys(urls))
|
||
|
||
|
||
def _slug(name: str) -> str:
|
||
return "_".join(normalized(name).split())
|
||
|
||
|
||
def wiki_items(body: str, urls: list[str]) -> tuple[list[str], list[dict], list[str], list[str]]:
|
||
"""(head blocks, one item per person, tail blocks, links not tied to a person) of a wiki answer."""
|
||
blocks = [b.strip() for b in re.split(r"\n\s*\n", body) if b.strip()]
|
||
head, items, tail = [], [], []
|
||
current = None
|
||
for block in blocks:
|
||
first = block.split("\n", 1)[0]
|
||
heading = HEADING.match(first)
|
||
name = re.split(r"\s+—\s+", heading.group(1))[0].strip() if heading else ""
|
||
if tail or TAIL.match(block):
|
||
tail.append(block)
|
||
elif heading and PERSON_NAME.fullmatch(name) and not name.endswith(":"):
|
||
if current is not None and current["name"] == name:
|
||
current["blocks"].append(block) # the same person's next work
|
||
else:
|
||
current = {"name": name, "school": WIKI_SCHOOL, "source": "wiki", "blocks": [block]}
|
||
items.append(current)
|
||
elif current is None:
|
||
head.append(block)
|
||
else:
|
||
current["blocks"].append(block)
|
||
general = []
|
||
for url in urls:
|
||
owner = next((i for i in items if _slug(i["name"]) in url), None)
|
||
if owner is not None:
|
||
owner.setdefault("urls", []).append(url)
|
||
else:
|
||
general.append(url)
|
||
for item in items:
|
||
item["text"] = "\n\n".join(item.pop("blocks"))
|
||
item.setdefault("urls", [])
|
||
return head, items, tail, general
|
||
|
||
|
||
def render_item(number: int, item: dict, mixed: bool) -> str:
|
||
if item["source"] == "topic":
|
||
return f"**{number}. {item['name']}** — {item['text']}"
|
||
if item["source"] == "crzp":
|
||
return f"**{number}. {item['name']}** (CRZP) — {item['text']}"
|
||
first, _, rest = item["text"].partition("\n")
|
||
heading = HEADING.match(first)
|
||
title = heading.group(1) if heading else item["name"]
|
||
line = f"**{number}. {title}**" + (" (ZP Wiki)" if mixed else "")
|
||
return line + ("\n" + rest if rest else "")
|
||
|
||
|
||
def merge_people(wiki: list[dict], crzp_items: list[dict]) -> list[dict]:
|
||
"""A CRZP work of a person already in the ZP Wiki list joins that person's item (no person twice)."""
|
||
by_name = {normalized(item["name"]): item for item in wiki}
|
||
rest = []
|
||
for item in crzp_items:
|
||
owner = by_name.get(normalized(item["name"]))
|
||
if owner is not None and item.get("school") == TUKE:
|
||
owner["text"] += f"\n- V CRZP: {item['text'].split(': ', 1)[-1]}"
|
||
owner.setdefault("urls", []).extend(item.get("urls", []))
|
||
else:
|
||
rest.append(item)
|
||
return rest
|
||
|
||
|
||
def first_page(wiki: list[dict], crzp_items: list[dict]) -> tuple[list[dict], list[dict]]:
|
||
"""(shown, rest): ZP Wiki people first (at most WIKI_FIRST when CRZP has works too), CRZP fills the page."""
|
||
crzp_items = merge_people(wiki, crzp_items)
|
||
take_wiki = min(len(wiki), WIKI_FIRST if crzp_items else PAGE)
|
||
take_crzp = min(len(crzp_items), PAGE - take_wiki)
|
||
take_wiki = min(len(wiki), PAGE - take_crzp)
|
||
shown = wiki[:take_wiki] + crzp_items[:take_crzp]
|
||
return shown, wiki[take_wiki:] + crzp_items[take_crzp:]
|
||
|
||
|
||
def more_line(rest: list[dict]) -> str:
|
||
if all(i["source"] == "topic" for i in rest):
|
||
word = "ďalšia téma" if len(rest) == 1 else "ďalšie témy" if len(rest) < 5 else "ďalších tém"
|
||
return f"Našl{'a' if len(rest) == 1 else 'i' if len(rest) < 5 else 'o'} sa ešte {len(rest)} {word}. Chceš vidieť ďalšie?"
|
||
wiki = sum(1 for i in rest if i["source"] == "wiki")
|
||
crzp_count = len(rest) - wiki
|
||
parts = [p for p in (f"{wiki} v ZP Wiki" if wiki else "", f"{crzp_count} v CRZP" if crzp_count else "") if p]
|
||
word = "ďalšia osoba" if len(rest) == 1 else "ďalšie osoby" if len(rest) < 5 else "ďalších osôb"
|
||
return f"Našl{'a' if len(rest) == 1 else 'i' if len(rest) < 5 else 'o'} sa ešte {len(rest)} {word} ({', '.join(parts)}). Chceš vidieť ďalšie?"
|
||
|
||
|
||
def compose_list(head: list[str], shown: list[dict], rest: list[dict], tail: list[str], general_urls: list[str],
|
||
*, start: int = 1, mixed: bool) -> tuple[str, list[str], str | None]:
|
||
"""(text, sources, id of the stored rest)."""
|
||
blocks = list(head) + [render_item(start + i, item, mixed) for i, item in enumerate(shown)] + list(tail)
|
||
urls = list(dict.fromkeys([*general_urls, *[u for item in shown for u in item.get("urls", [])]]))
|
||
if urls:
|
||
blocks.append("Zdroje: " + ", ".join(urls))
|
||
from tools.answers import pages
|
||
page_id = pages.put(rest, {"mixed": mixed}) if rest else None
|
||
if rest and page_id:
|
||
blocks.append(more_line(rest))
|
||
return "\n\n".join(blocks), urls, page_id
|
||
|
||
|
||
def entity(item: dict, number: int) -> dict:
|
||
return {k: v for k, v in {"n": number, "name": item["name"], "school": item.get("school"),
|
||
"source": item.get("source"), "sid": item.get("sid"), "title": item.get("title")}.items()
|
||
if v is not None}
|
||
|
||
|
||
# ------------------------------------------------------------------ composition
|
||
|
||
def _ok(question, engine, text, urls, base) -> dict:
|
||
keep = {k: v for k, v in base.items() if k not in ("answer", "status", "reason", "delivery_check",
|
||
"source_urls", "conversation_context")}
|
||
return {**keep, "status": "ok", "reason": None, "accepted_for_delivery": True, "answer": text,
|
||
"source_urls": list(dict.fromkeys(urls)), "claims": base.get("claims", []),
|
||
"index_version": base.get("index_version"), "complete": base.get("complete", True),
|
||
"truncated": base.get("truncated", False), "truncation_reasons": base.get("truncation_reasons", []),
|
||
"delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": fingerprint(question, engine)}}
|
||
|
||
|
||
def _with_token(result: dict, *, wiki_context=None, entities=(), options=(), question="", crzp=None,
|
||
primary="wiki", page=None) -> dict:
|
||
wiki_context = wiki_context or {}
|
||
token = seal({"wiki": wiki_context.get("token"), "entities": list(entities)[:40], "options": list(options)[:6],
|
||
"question": question, "crzp": crzp, "primary": primary, "page": page})
|
||
if token:
|
||
result["conversation_context"] = {**{k: v for k, v in wiki_context.items() if k != "token"},
|
||
"token": token, "work_options": wiki_context.get("work_options", []),
|
||
"meeting_options": wiki_context.get("meeting_options", [])}
|
||
return result
|
||
|
||
|
||
NEXT_PAGE = re.compile(r"\b(?:dalsi\w*|dalej|viac|ostatn\w*|zvysn\w*|pokracuj\w*|zvysok)\b")
|
||
YES = re.compile(r"^\W*(?:ano|jasne|ok|okej|hej|chcem|daj|ukaz|prosim|urcite|samozrejme)(?:\s+(?:ano|prosim|daj|ukaz|ich|mi|ho|aj))*\W*$")
|
||
ORDINAL_WORDS = {"prv": 1, "druh": 2, "tret": 3, "stvrt": 4, "piat": 5, "siest": 6, "siedm": 7, "osm": 8, "deviat": 9,
|
||
"desiat": 10}
|
||
REFERENCE = re.compile(
|
||
r"(?i)\b(?:(?:t[eé]n|t[aá]|t[uú]|toho|tej|tomu|tom|tou)\s+)?"
|
||
r"(prv|druh|tret|štvrt|stvrt|piat|šiest|siest|siedm|ôsm|osm|deviat|desiat|posledn)\w*"
|
||
r"(?:\s+(?:z\s+nich|zo\s+zoznamu|v\s+poradí|v\s+poradi|študent\w*|student\w*|osob\w*|človek\w*|clovek\w*))"
|
||
r"|\b(?:číslo|cislo|č\.)\s*(\d{1,2})\b")
|
||
|
||
|
||
def _reference(question: str, entities: list[dict]) -> tuple[re.Match, dict] | None:
|
||
""""prvý z nich", "toho tretieho študenta", "číslo 2" -> the shown person it points at."""
|
||
match = REFERENCE.search(question)
|
||
if not match or not entities:
|
||
return None
|
||
if match.group(2):
|
||
number = int(match.group(2))
|
||
else:
|
||
word = normalized(match.group(1))
|
||
number = len(entities) if word.startswith("posledn") else next(
|
||
(n for stem, n in ORDINAL_WORDS.items() if word.startswith(stem)), None)
|
||
chosen = next((e for e in entities if e.get("n") == number), None)
|
||
return (match, chosen) if chosen else None
|
||
|
||
|
||
def _stamp(result: dict, question: str, engine: str) -> dict:
|
||
"""An answer prepared for a question this layer rewrote ("prvý z nich" -> the name, "2" -> the
|
||
pending question about the chosen work) is bound to the message the user sent. Any other
|
||
fingerprint stays as it is, so the client still rejects an answer that belongs elsewhere."""
|
||
internal = result.pop("_asked_as", None) or []
|
||
check = result.get("delivery_check")
|
||
if isinstance(check, dict) and check.get("request_fingerprint") in {fingerprint(q, engine) for q in internal}:
|
||
result["delivery_check"] = {**check, "request_fingerprint": fingerprint(question, engine)}
|
||
return result
|
||
|
||
|
||
_warmed = threading.Event()
|
||
|
||
|
||
def answer(question: str, *, engine: str, limit: int = 5, context_token: str | None = None, complete=None,
|
||
deliver=None) -> dict:
|
||
"""deliver: the ZP Wiki answer function (the API passes its own, tests replace it)."""
|
||
question = question.strip()
|
||
if enabled() and not _warmed.is_set(): # tag rarity for related topics, counted once in the background
|
||
_warmed.set()
|
||
_pool().submit(tag_frequencies)
|
||
return _stamp(_answer(question, engine=engine, limit=limit, context_token=context_token, complete=complete,
|
||
deliver=deliver), question, engine)
|
||
|
||
|
||
def _ask_choice(question: str, engine: str, previous: dict) -> dict:
|
||
options = previous["options"]
|
||
text = ("Odpovedz prosím číslom: " + ", ".join(f"{i} – {o['label']}" for i, o in enumerate(options, 1)) + ".")
|
||
result = _ok(question, engine, text, [], {})
|
||
result.update(status="ambiguous", reason="clarification_required",
|
||
delivery_check={"layer": "planning", "code": "clarification_required",
|
||
"request_fingerprint": fingerprint(question, engine)})
|
||
return _with_token(result, options=options, question=previous["question"])
|
||
|
||
|
||
def _next_page(question: str, engine: str, previous: dict) -> dict:
|
||
from tools.answers import pages
|
||
stored = pages.get(previous.get("page"))
|
||
entities = previous.get("entities") or []
|
||
if not stored:
|
||
return _ok(question, engine, "Ďalšie položky predchádzajúcej odpovede už nie sú k dispozícii; zopakuj otázku.",
|
||
[], {})
|
||
items, mixed = stored["items"], stored["meta"].get("mixed", False)
|
||
start = max((e.get("n", 0) for e in entities), default=0) + 1
|
||
text, urls, page_id = compose_list([], items[:PAGE], items[PAGE:], [], [], start=start, mixed=mixed)
|
||
shown = entities + [entity(item, start + i) for i, item in enumerate(items[:PAGE])]
|
||
return _with_token(_ok(question, engine, text, urls, {}), entities=shown, question=question, page=page_id,
|
||
crzp=previous.get("crzp"), primary=previous.get("primary", "wiki"))
|
||
|
||
|
||
def _schools_answer(question: str, engine: str, previous: dict) -> dict:
|
||
entities = previous["entities"]
|
||
lines = [f"{e.get('n', i)}. {e['name']} — {e.get('school') or 'škola neuvedená'}"
|
||
for i, e in enumerate(entities, 1)]
|
||
text = "Školy ľudí z predchádzajúcej odpovede:\n\n" + "\n".join(lines)
|
||
return _with_token(_ok(question, engine, text, [], {}), entities=entities, question=question,
|
||
page=previous.get("page"), crzp=previous.get("crzp"), primary=previous.get("primary", "wiki"))
|
||
|
||
|
||
def _crzp_work_answer(question: str, engine: str, chosen: dict, complete=None) -> dict:
|
||
"""A follow-up about a CRZP work of the previous list ("o čom je práca tretieho z nich?")."""
|
||
from tools.crzp.answering import answer as crzp
|
||
from tools.crzp.answering.understand import Person, understand
|
||
query = understand(question, complete=complete)
|
||
query.people = [Person(name=chosen["name"], role="author", typed=chosen["name"])]
|
||
query.sids, query.title, query.topics, query.alternatives = [chosen["sid"]], None, [], []
|
||
query.school = query.unit = None
|
||
query.from_model = True
|
||
if query.intent in ("list", "similar", "schools"):
|
||
query.intent = "about"
|
||
result = crzp._answer_query(question, query, complete)
|
||
for key in ("_shown", "_asked", "_options", "_entities"):
|
||
result.pop(key, None)
|
||
if result.get("accepted_for_delivery") is not True:
|
||
return result
|
||
body, urls = split_sources(clean_crzp(result["answer"]))
|
||
urls = list(dict.fromkeys([*urls, *(result.get("source_urls") or [])]))
|
||
text = body + ("\n\nZdroje: " + ", ".join(urls) if urls else "")
|
||
return {**result, "answer": text, "source_urls": urls,
|
||
"delivery_check": {**result["delivery_check"], "request_fingerprint": fingerprint(question, engine)}}
|
||
|
||
|
||
def _answer(question: str, *, engine: str, limit: int, context_token: str | None, complete, deliver) -> dict:
|
||
from tools.answers.answer_delivery import deliver_answer as default_deliver
|
||
deliver_answer = deliver or default_deliver
|
||
from tools.crzp.answering.wiki_bridge import wiki_match, wiki_person
|
||
previous = unseal(context_token)
|
||
if previous and previous.get("error"):
|
||
from tools.answers.answer_errors import public_failure
|
||
failure = public_failure({"reason": previous["error"], "engine": engine})
|
||
failure["delivery_check"] = {"layer": "context", "code": "unavailable",
|
||
"request_fingerprint": fingerprint(question, engine)}
|
||
return failure
|
||
wiki_token = previous.get("wiki") if previous else context_token
|
||
low = normalized(question)
|
||
if previous and previous.get("options"): # "Ktorú prácu myslíš?" was asked
|
||
chosen = _choice(low, previous["options"])
|
||
if chosen is not None:
|
||
result = _answer_choice(previous["question"], chosen, engine=engine, limit=limit, complete=complete,
|
||
deliver=deliver_answer)
|
||
result["_asked_as"] = [previous["question"], *result.pop("_asked_as", [])]
|
||
if result.get("accepted_for_delivery") is True and result.get("status") == "ok":
|
||
# the offer stays: "a tú druhú?" / "1" after "2" answers the same question for the other work
|
||
result = _with_token(result, wiki_context=result.get("conversation_context"),
|
||
options=previous["options"], question=previous["question"])
|
||
return result
|
||
if len(low.split()) <= 3 and not wiki_person(question):
|
||
return _ask_choice(question, engine, previous)
|
||
wants_more = bool(YES.match(low) or NEXT_PAGE.search(low) and len(low.split()) <= 6)
|
||
if previous and previous.get("page") and wants_more:
|
||
return _next_page(question, engine, previous)
|
||
if previous and previous.get("entities") and wants_more: # nothing more was offered
|
||
return _with_token(_ok(question, engine, "Všetko nájdené už je v predchádzajúcej odpovedi.", [], {}),
|
||
entities=previous["entities"], question=question, crzp=previous.get("crzp"),
|
||
primary=previous.get("primary", "wiki"))
|
||
entities = (previous or {}).get("entities") or []
|
||
if entities and SCHOOLS.search(low) and not wiki_person(question):
|
||
return _schools_answer(question, engine, previous)
|
||
asked = question
|
||
reference = _reference(question, entities) if len(entities) > 1 else None
|
||
if reference:
|
||
match, chosen = reference
|
||
if chosen.get("source") == "crzp" and chosen.get("sid"):
|
||
return _crzp_work_answer(question, engine, chosen, complete)
|
||
question = (question[:match.start()] + chosen["name"] + question[match.end():]).strip()
|
||
low, wiki_token, previous = normalized(question), None, None # a question about that person now
|
||
return {**_answer(question, engine=engine, limit=limit, context_token=None, complete=complete,
|
||
deliver=deliver), "_asked_as": [question]}
|
||
match = wiki_match(question)
|
||
if previous and previous.get("primary") == "crzp" and previous.get("crzp") and not match:
|
||
# The last answer came from CRZP ("O čom je práca Jána Nováka?" -> "2", "a kto bol oponent?").
|
||
from tools.crzp.answering import answer as crzp
|
||
followed = crzp._answer(question, complete, context_token=previous["crzp"])
|
||
rows = followed.pop("_shown", None) or []
|
||
followed.pop("_asked", None), followed.pop("_options", None)
|
||
shown = followed.pop("_entities", None) or []
|
||
if followed.get("accepted_for_delivery") is True:
|
||
body, urls = split_sources(clean_crzp(followed["answer"]))
|
||
urls = list(dict.fromkeys([*urls, *(followed.get("source_urls") or [])]))[:12]
|
||
result = {**followed, "answer": body + ("\n\nZdroje: " + ", ".join(urls) if urls else ""),
|
||
"delivery_check": {**followed["delivery_check"], "request_fingerprint": fingerprint(question, engine)}}
|
||
token = (followed.get("conversation_context") or {}).get("token")
|
||
result.pop("conversation_context", None)
|
||
entities = [{**e, "n": i} for i, e in enumerate(shown, 1)] or previous.get("entities", [])
|
||
return _with_token(result, entities=entities, question=question, crzp=token or previous["crzp"],
|
||
primary="crzp")
|
||
person, student = (match["name"], match["student"]) if match else (None, True)
|
||
started = time.monotonic()
|
||
pool = _pool()
|
||
from tools.answers import timing
|
||
import contextvars
|
||
|
||
def timed(name, function, *args, **kwargs):
|
||
began = time.monotonic()
|
||
try:
|
||
return function(*args, **kwargs)
|
||
finally:
|
||
timing.step(name, time.monotonic() - began)
|
||
|
||
wiki_future = pool.submit(contextvars.copy_context().run, timed, "wiki", deliver_answer, question, engine=engine,
|
||
limit=limit,
|
||
**({"context_token": wiki_token} if wiki_token else {}))
|
||
crzp_future = pool.submit(contextvars.copy_context().run, timed, "crzp", crzp_part, question, person,
|
||
complete=complete, student=student) if enabled() else None
|
||
wiki = wiki_future.result()
|
||
try:
|
||
part = crzp_future.result(timeout=max(1.0, CRZP_SECONDS - (time.monotonic() - started))) if crzp_future else None
|
||
except FutureTimeout:
|
||
part = None # CRZP too slow this time: the wiki answer is not held back (its search stays cached)
|
||
except Exception:
|
||
part = None
|
||
return compose(asked, question, engine, wiki, part, person)
|
||
|
||
|
||
def compose(asked: str, question: str, engine: str, wiki: dict, part: dict | None, person: str | None) -> dict:
|
||
"""One answer from the ZP Wiki result and the CRZP part."""
|
||
from tools.answers.finalize import finalize
|
||
low = normalized(question)
|
||
if wiki.get("reason") in REFUSED or wiki.get("status") == "ambiguous" and (
|
||
person or wiki.get("reason") not in WIKI_CLARIFICATION_YIELDS):
|
||
part = None # refused/out of scope, or the wiki asks first (which person, which work): no CRZP
|
||
wiki_ok = wiki.get("accepted_for_delivery") is True and wiki.get("status") == "ok"
|
||
if part is not None and part["mode"] == "explicit" and wiki.get("status") == "ambiguous":
|
||
wiki_ok = False
|
||
if wiki.get("accepted_for_delivery") is True and isinstance(wiki.get("answer"), str):
|
||
wiki = {**wiki, "answer": finalize(wiki["answer"], wiki.get("source_urls"))}
|
||
wiki_context = wiki.get("conversation_context")
|
||
if part is None or wiki.get("status") == "ambiguous" and part["mode"] != "explicit":
|
||
# (a wiki clarification such as "Uveď meno študenta" yields to a question CRZP answers as a whole)
|
||
if wiki.get("accepted_for_delivery") is not True:
|
||
return wiki
|
||
if wiki_ok:
|
||
body, urls = split_sources(wiki["answer"])
|
||
head, items, tail, general = wiki_items(body, urls or wiki.get("source_urls") or [])
|
||
if len(items) >= 2:
|
||
shown, rest = first_page(items, [])
|
||
text, shown_urls, page_id = compose_list(head, shown, rest, tail, general, mixed=False)
|
||
result = {**wiki, "answer": text, "source_urls": shown_urls}
|
||
return _with_token(result, wiki_context=wiki_context, question=question, page=page_id,
|
||
entities=[entity(item, i) for i, item in enumerate(shown, 1)])
|
||
return _with_token(wiki, wiki_context=wiki_context, question=question)
|
||
# Different works in the two sources and the question names none: ask which one.
|
||
if person and part["mode"] == "person" and (METHODS.search(low) or ABOUT.search(low)):
|
||
options = _work_options(person, part, low)
|
||
if options:
|
||
text = (f"{person} má v podkladoch viac prác. Ktorú myslíš?\n\n" +
|
||
"\n".join(f"{i}. {o['label']} ({'ZP Wiki' if o['source'] == 'wiki' else 'CRZP'})"
|
||
for i, o in enumerate(options, 1)) + "\n\nOdpovedz číslom alebo typom práce.")
|
||
result = _ok(question, engine, text, [], wiki)
|
||
result.update(status="ambiguous", reason="clarification_required",
|
||
delivery_check={"layer": "planning", "code": "clarification_required",
|
||
"request_fingerprint": fingerprint(question, engine)})
|
||
return _with_token(result, wiki_context=wiki_context, options=options, question=question)
|
||
if part["mode"] == "explicit" and part.get("status") == "ambiguous": # CRZP asks which namesake
|
||
result = _ok(question, engine, part["text"], [], wiki if wiki_ok else {"index_version": wiki.get("index_version")})
|
||
result.update(status="ambiguous", reason="clarification_required",
|
||
delivery_check={"layer": "planning", "code": "clarification_required",
|
||
"request_fingerprint": fingerprint(question, engine)})
|
||
return _with_token(result, question=question, crzp=part.get("crzp_token"), primary="crzp")
|
||
base = wiki if wiki_ok else {"index_version": wiki.get("index_version")}
|
||
wiki_body, wiki_urls = split_sources(wiki["answer"]) if wiki_ok else ("", [])
|
||
wiki_urls = wiki_urls or (wiki.get("source_urls") or [] if wiki_ok else [])
|
||
if part["mode"] == "related": # "Ktoré témy súvisia s X?": the topics and who did them
|
||
own = wiki_people_for(part["topic"]) if part.get("topic") else []
|
||
head = [f"Témy súvisiace s témou „{part['topic']}“ (vyskytujú sa v tých istých prácach):"]
|
||
if own:
|
||
head.append(f"Samotnej téme „{part['topic']}“ sa v ZP Wiki venujú: {', '.join(own[:8])}"
|
||
+ (f" a ďalší ({len(own) - 8})." if len(own) > 8 else "."))
|
||
shown, rest = part["items"][:PAGE], part["items"][PAGE:]
|
||
text, urls, page_id = compose_list(head, shown, rest, [], [], mixed=False)
|
||
return _with_token(_ok(question, engine, text, urls, base), wiki_context=wiki_context, question=question,
|
||
entities=[entity(item, i) for i, item in enumerate(shown, 1)], page=page_id)
|
||
if part.get("items") is not None: # topic or similar works: one numbered list with the wiki people
|
||
head, items, tail, general = wiki_items(wiki_body, wiki_urls) if wiki_ok else ([], [], [], [])
|
||
if wiki_ok and not items: # a wiki answer that is no list stays whole, the CRZP works follow it
|
||
head, tail, general = [wiki_body], [], wiki_urls
|
||
if not wiki_ok:
|
||
head = ["V ZP Wiki k tomu nie je záznam. Práce na túto tému v CRZP (Centrálny register záverečných prác):"]
|
||
shown, rest = first_page(items, part["items"])
|
||
text, urls, page_id = compose_list(head, shown, rest, tail, general, mixed=True)
|
||
return _with_token(_ok(question, engine, text, urls, base), wiki_context=wiki_context, question=question,
|
||
entities=[entity(item, i) for i, item in enumerate(shown, 1)], page=page_id)
|
||
crzp_body, crzp_urls = split_sources(part["text"])
|
||
crzp_urls = list(dict.fromkeys([*crzp_urls, *part.get("urls", [])]))[:12]
|
||
entities = [{**e, "n": i} for i, e in enumerate(part.get("entities") or [], 1)]
|
||
if not wiki_ok and (part["mode"] == "explicit" or wiki.get("status") == "error"):
|
||
text = crzp_body
|
||
elif wiki_ok:
|
||
text = wiki_body + "\n\nV CRZP (Centrálny register záverečných prác):\n\n" + crzp_body
|
||
else:
|
||
text = "V ZP Wiki k tomu nie je záznam.\n\nV CRZP (Centrálny register záverečných prác):\n\n" + crzp_body
|
||
urls = list(dict.fromkeys([*wiki_urls, *crzp_urls]))
|
||
if urls:
|
||
text += "\n\nZdroje: " + ", ".join(urls)
|
||
result = _ok(question, engine, text, urls, base)
|
||
return _with_token(result, wiki_context=wiki_context, entities=entities, question=question,
|
||
crzp=part.get("crzp_token"), primary="crzp" if part["mode"] == "explicit" and not wiki_ok else "wiki")
|
||
|
||
|
||
def _work_options(person: str, part: dict, low: str) -> list[dict]:
|
||
"""The person's works when the two sources hold works of different types and the question names none."""
|
||
if re.search(r"\b(?:bakalar\w*|diplom\w*|dizerta\w*|projekt\w*)\b", low):
|
||
return []
|
||
wiki = wiki_works(person)
|
||
wiki_types = {w["type"] for w in wiki.values()}
|
||
thesis = {"diploma_thesis": "diplom", "bachelor_thesis": "bakalar"}
|
||
crzp_only = [w for w in part.get("works", []) if w["type"] in thesis and w["type"] not in wiki_types]
|
||
if not crzp_only:
|
||
return []
|
||
wiki_options = [{"source": "wiki", "label": label + (f" „{w['topic']}“" if w.get("topic") else ""), "work": label}
|
||
for label, w in wiki.items() if w["type"] in thesis][:2]
|
||
return wiki_options + crzp_only if wiki_options else []
|
||
|
||
|
||
CHOICE_FILLERS = {"tak", "teda", "ta", "tu", "ten", "to", "tej", "chcem", "daj", "prosim", "cislo", "moznost", "a", "ano",
|
||
"no", "hej", "mi", "tu", "pracu", "praca", "prace", "o", "tej", "tom", "volba", "ok", "jasne", "tú"}
|
||
NUMBER_WORDS = {"jedna": 1, "jednotku": 1, "jednotka": 1, "prvu": 1, "prva": 1, "prvy": 1, "prve": 1, "prvej": 1,
|
||
"dva": 2, "dve": 2, "dvojku": 2, "dvojka": 2, "druhu": 2, "druha": 2, "druhy": 2, "druhe": 2, "druhej": 2,
|
||
"tri": 3, "trojku": 3, "trojka": 3, "tretiu": 3, "tretia": 3, "treti": 3, "tretej": 3}
|
||
|
||
|
||
def _choice(low: str, options: list[dict]) -> dict | None:
|
||
""""2", "tak 1", "tú prvú", "jednotku", "bakalársku", "z wiki" -> the chosen option."""
|
||
words = [w for w in re.findall(r"[a-z0-9]+", low) if w not in CHOICE_FILLERS]
|
||
if len(words) == 1:
|
||
word = words[0]
|
||
number = int(word) if word.isdigit() else NUMBER_WORDS.get(word)
|
||
if number is not None:
|
||
return options[number - 1] if 1 <= number <= len(options) else None
|
||
if len(words) <= 4 and CHOICE_CRZP.search(low):
|
||
return next((o for o in options if o["source"] == "crzp"), None)
|
||
if len(words) <= 4 and CHOICE_WIKI.search(low):
|
||
return next((o for o in options if o["source"] == "wiki"), None)
|
||
return None
|
||
|
||
|
||
def _answer_choice(question: str, chosen: dict, *, engine: str, limit: int, complete=None, deliver=None) -> dict:
|
||
"""The answer to the pending question about the chosen work (the caller binds it to the reply)."""
|
||
from tools.answers.finalize import finalize
|
||
deliver_answer = deliver
|
||
from tools.crzp.answering.wiki_bridge import wiki_person
|
||
if chosen["source"] == "wiki":
|
||
work = normalized(chosen["work"])
|
||
adjective = "diplomovej" if work.startswith("diplom") else "bakalárskej" if work.startswith("bakal") else ""
|
||
asked = re.sub(r"\b(svojej\s+|jeho\s+|jej\s+)?(práci|práce|prácu)\b", lambda m: (m.group(1) or "") + adjective + " " + m.group(2),
|
||
question, count=1) if adjective else question
|
||
if asked == question:
|
||
asked = question.rstrip(" ?.") + f" ({chosen['work']})?"
|
||
result = deliver_answer(asked, engine=engine, limit=limit)
|
||
if result.get("accepted_for_delivery") is True and isinstance(result.get("answer"), str):
|
||
result = {**result, "answer": finalize(result["answer"], result.get("source_urls"))}
|
||
return {**result, "_asked_as": [asked]}
|
||
person = wiki_person(question)
|
||
part = crzp_part(question, person, complete=complete)
|
||
rows = [r for r in (part or {}).get("rows", []) if chosen.get("key") and _key_of(r) == chosen["key"]]
|
||
if not part or not rows:
|
||
return _ok(question, engine, "Vybranú prácu sa v CRZP nepodarilo znova načítať. Skús otázku zopakovať.", [], {})
|
||
from tools.crzp.answering import answer as crzp
|
||
row = rows[0]
|
||
detail = crzp.fetch.detail(row["sid"]) if row.get("sid") else None
|
||
low = normalized(question)
|
||
if METHODS.search(low):
|
||
text = crzp_methods(person, row, detail)
|
||
else:
|
||
label, summary = crzp.summary(detail, complete) if detail else ("", "")
|
||
text = f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{summary}"
|
||
urls = [row["url"]] if row.get("url") else []
|
||
return _ok(question, engine, text + ("\n\nZdroj: " + urls[0] if urls else ""), urls, {})
|
||
|
||
|
||
def _key_of(row: dict) -> str:
|
||
from tools.crzp.answering import answer as crzp
|
||
return crzp.work_key(row)
|