"""ZP Wiki and CRZP in one answer for /rag and /graphrag. Every question is answered from ZP Wiki (the normal RAG / GraphRAG path) and, at the same time, from CRZP when CRZP can add something (see crzp_plan): - a person ZP Wiki knows: that person's works in CRZP at Technická univerzita v Košiciach (supervisor, opponent, abstract, keywords the wiki may lack); - a topic or "similar works": CRZP works on the same topic, with their schools; - a question naming CRZP, another school, opponents or defended works of a department: CRZP is the main source. Meetings, tasks, progress and results stay ZP Wiki only. The answer is one text: a list of people is one numbered list (ZP Wiki people first, then CRZP works, each marked with its source), the first PAGE items are shown and the rest is offered ("Chceš vidieť ďalších?"); all sources are in one line at the end. A signed token carries the wiki's own token, the shown people (for "a z akých škôl sú?", "a čo robil prvý z nich?") and the id of the not yet shown items (tools/answers/pages.py). """ from __future__ import annotations import base64 import hashlib import hmac import json import os import re import threading import time from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout from tools.answers.context_policy import normalized from tools.answers.delivery_contract import fingerprint CRZP_SECONDS = float(os.getenv("CRZP_COMBINED_SECONDS", "45")) LIFETIME = 24 * 3600 PREFIX = "zc1." # plain JSON body (still read) PREFIX2 = "zc2." # zlib-compressed JSON body TUKE = "Technická univerzita v Košiciach" WIKI_SCHOOL = "Technická univerzita v Košiciach (KEMT FEI)" # What only the wiki records: CRZP is not asked. WIKI_ONLY = re.compile(r"\b(?:stretnut\w*|pokrok\w*|progres\w*|uloh\w*|zadan\w*|plan\w*|termin\w*|tyzd\w*|" r"repozitar\w*|git\w*|stav\w*|poznam\w*|konzult\w*|zaciat\w*|studi\w*|obhaj\w*|ukonc\w*|" r"dokonc\w*|uspes\w*|statnic\w*|hodnot\w*|znamk\w*|problem\w*|vtedy|posledn\w*)\b") EXPLICIT = re.compile(r"\b(?:crzp|registr\w*|register|celoslovensk\w*|na slovensku|(?:inych|inej|inom|ostatnych|" r"vsetkych|dalsich)\s+(?:vysok\w+\s+)?(?:skol|univerzit|fakult)\w*|mimo tuke|oponent\w*|oponoval\w*)\b") METHODS = re.compile(r"\b(?:technolog\w*|nastroj\w*|metod\w*|pouziv\w*|pouzil\w*|vyuziv\w*|vyuzil\w*|framework\w*)\b") ABOUT = re.compile(r"\b(?:o com|zhrn\w*|abstrakt\w*|ciel\w*)\b") # not "čomu sa venuje": the current work # A question about a thesis (not "na čom pracuje", which CRZP's submitted works cannot answer). THESIS = re.compile(r"\b(?:prac(?:a|e|u|i|ou|ach|ami)?|bakalar\w*|diplom\w*|dizerta\w*|habilita\w*|tem(?:a|u|y|ou)|" r"oponent\w*|vedu\w*|skolitel\w*|abstrakt\w*|klucov\w*|tagy?)\b") WIKI_CLARIFICATION_YIELDS = {"missing_subject", "unsupported_content_question", "unsupported_constraint"} # "Koľko prác obhájili na KEMT v roku 2024?": defended works of a school or department are CRZP's # published works, not a wiki question about one student's defence. DEFENDED = re.compile(r"\b(?:obhaj\w*|obhajen\w*|obhajil\w*|uspesn\w*)\b") TOPIC_MIN_SCORE = float(os.getenv("CRZP_TOPIC_MIN_SCORE", "0")) PAGE = int(os.getenv("ANSWER_PAGE_SIZE", "6")) WIKI_FIRST = 4 # a mixed list shows at most this many ZP Wiki people before the CRZP works REFUSED = {"request_out_of_scope", "irrelevant", "unsupported_request", "out_of_scope"} RELATED = re.compile(r"\b(?:suvis\w*|pribuzn\w*)\b") SIMILAR = re.compile(r"\b(?:podobn\w*|suvisiac\w*|pribuzn\w*|spolupra\w*|rovnak\w* tem\w*)\b") TOPIC_LIST = re.compile(r"^\W*(?:kto|ktor\w*|ak\w*)\b") SCHOOLS = re.compile(r"\b(?:z|zo|na) (?:akych|akej|ktorych|ktorej) (?:skol|univerzit|fakult)\w*|\bkde (?:studoval\w*|studuj\w*)" r"|\b(?:ktor|ak)\w* (?:skol|univerzit)\w*") CHOICE_WIKI = re.compile(r"\b(?:wiki|diplom\w*|dp)\b") CHOICE_CRZP = re.compile(r"\b(?:crzp|bakalar\w*|bp)\b") _shared = {"pool": None} _shared_lock = threading.Lock() def _pool() -> ThreadPoolExecutor: """One pool for all questions (no new threads per request).""" with _shared_lock: if _shared["pool"] is None: _shared["pool"] = ThreadPoolExecutor(max_workers=int(os.getenv("ANSWER_COMBINED_WORKERS", "6")), thread_name_prefix="answer") return _shared["pool"] def enabled() -> bool: return os.getenv("ANSWER_WITH_CRZP", "1") != "0" # ------------------------------------------------------------------ token def _key() -> bytes | None: secret = os.getenv("SEARCH_API_KEY", "") return hashlib.sha256(("zp-combined-v1\0" + secret).encode()).digest() if secret else None def _b64(value: bytes) -> str: return base64.urlsafe_b64encode(value).decode().rstrip("=") def seal(payload: dict) -> str | None: """A signed, compressed token (the wiki's and CRZP's own tokens inside make it long).""" import zlib key = _key() if key is None: return None raw = json.dumps({**payload, "expires": int(time.time()) + LIFETIME}, ensure_ascii=False, separators=(",", ":")) body = _b64(zlib.compress(raw.encode(), 9)) token = PREFIX2 + body + "." + _b64(hmac.digest(key, body.encode(), "sha256")) return token if len(token) <= 16000 else None def unseal(token: str | None) -> dict | None: """The combined context, or None for a token that is not ours (a plain wiki token).""" import zlib if not isinstance(token, str) or not token.startswith((PREFIX, PREFIX2)) or _key() is None: return None try: body, signature = token[4:].split(".") expected = hmac.digest(_key(), body.encode(), "sha256") if not hmac.compare_digest(base64.b64decode(signature + "=" * (-len(signature) % 4), altchars=b"-_"), expected): raise ValueError data = base64.b64decode(body + "=" * (-len(body) % 4), altchars=b"-_") value = json.loads(zlib.decompress(data) if token.startswith(PREFIX2) else data) if value.get("expires", 0) < time.time(): return {"error": "expired_context"} return value except (ValueError, TypeError, UnicodeError, zlib.error): return {"error": "invalid_context"} # ------------------------------------------------------------------ wiki facts _index_cache: dict = {} _index_lock = threading.Lock() def _index(): """The published knowledge index, reloaded only when its version changes.""" from tools.knowledge.store import load_index from tools.rag.common import DB_FILE from tools.sync.index_state import read_version version = read_version(DB_FILE.parent) key = (version or {}).get("version") with _index_lock: if _index_cache.get("key") != key: _index_cache.update(key=key, index=load_index(DB_FILE.parent, version)) return _index_cache["index"] # "Kto je Ján Novák?", "Čo robil Peter Kováč?": a question about a person (for CRZP when the wiki does not know it). PERSON_Q = re.compile(r"^\W*(?:kto\s+(?:je|bol|bola|bude)|co\s+(?:robi|robil|robila|riesil|riesila)|na\s+com\s+pracoval\w*|" r"kde\s+studoval\w*|o\s+kom|povedz\s+mi\s+o)\b") HUMAN_NAME = re.compile(r"[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][a-záäčďéíĺľňóôŕšťúýž]+(?:-[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][a-záäčďéíĺľňóôŕšťúýž]+)?(?:\s+[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][a-záäčďéíĺľňóôŕšťúýž]+(?:-[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][a-záäčďéíĺľňóôŕšťúýž]+)?){1,2}") def COMPARE_TWO(low: str) -> bool: """"Mohol by A spolupracovať s B?", "Porovnaj A a B": two people named, a comparison of the pair.""" return bool(COMPARE.search(low)) and not TOPIC_LIST.match(low) COUNT_STUDENTS = re.compile(r"\bkolk\w*\s+(?:studentov|student\w*)\b") STUDY_START = re.compile(r"\b(?:zaciat\w*\s+stud\w*|zacal\w*\s+stud\w*)\b") COMPARE = re.compile(r"\b(?:porovn\w*|rozdiel\w*|odlis\w*|v com sa lis\w*|spolupr\w*)\b") # "a Ján Pták?", "a čo Ján Pták?", "a Ján Pták by mohol?": the previous question about another person. ELLIPSIS = re.compile(r"^\W*a(?:\s+(?:co|ako|kto|aj))?\s+\S") COULD = re.compile(r"\b(?:mohol|mohla|mohli|moze|mozu|hodil|hodila)\b") def _one_person(question: str) -> dict | None: """{"name", "input"}: the one ZP Wiki person a question names, with the words naming it.""" from tools.answers.question_plan import planning_catalog from tools.rag.common import DB_FILE from tools.retrieval.person_names import resolve_person_names try: resolution = resolve_person_names(question, planning_catalog(DB_FILE) or []) except Exception: return None matches = resolution.get("matches") or [] if resolution.get("status") != "ok" or len({m["name"] for m in matches}) != 1: return None return {"name": matches[0]["name"], "input": matches[0].get("input") or matches[0]["name"]} def ellipsis(question: str, previous: dict | None) -> str | None: """The full question behind a short follow-up about another person: "a Ján Pták?" after "Kto je vedúci práce Jakuba Schwarca?" -> "Kto je vedúci práce Ján Pták?"; "a Ján Pták by mohol?" after "Kto by mohol spolupracovať s Jakubom Schwarcom?" -> whether these two could work together.""" low = normalized(question) if not previous or not previous.get("question") or not ELLIPSIS.match(low) or len(low.split()) > 7: return None bare = re.sub(r"(?i)^\W*a(?:\s+(?:čo|co|ako|kto|aj))?\s+", "", question) bare = re.sub(r"(?i)\s+by\s+(?:mohol|mohla|mohli)\W*$|\W+$", "", bare) new, old = _one_person(bare) or _one_person(question), _one_person(previous["question"]) if not new and re.fullmatch(r"[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][\w\-]+(?:\s+[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][\w\-]+){1,2}", bare): new = {"name": bare, "input": bare} # a teacher is resolved only in a question ("Koho vedie Matúš Pleva?") if not new or not old or new["name"] == old["name"]: return None asked = previous["question"] if COULD.search(low) and SIMILAR.search(normalized(asked)): return f"Mohol by {new['name']} spolupracovať s osobou {old['name']}?" if old["input"] in asked: return asked.replace(old["input"], new["name"], 1) return None AGAIN_FILLERS = {"a", "ale", "no", "tak", "teda", "este", "presne", "vlastne", "konkretne", "mi", "povedz", "napis", "prosim"} def again(question: str, previous: dict | None) -> str | None: """"ale čo robil?" after "Čo robil Ján Pták v bakalárskej práci?": the user asks the previous question once more (every content word of the reply is in it, no person is named).""" if not previous or not previous.get("question") or previous.get("options"): return None words = [w for w in re.findall(r"[a-z0-9]+", normalized(question)) if w not in AGAIN_FILLERS] before = set(re.findall(r"[a-z0-9]+", normalized(previous["question"]))) if not words or len(words) > 4 or not set(words) <= before or _one_person(question): return None return previous["question"] def comparison_block(names: list[str]) -> str | None: """"Porovnanie": the most specific concepts both students document and those only one does (content graph, no model call), shown above the two students' records.""" try: from tools.knowledge.activities import load_graph, compare_people from tools.rag.common import DB_FILE index = _index() graph = load_graph(DB_FILE.parent, index) if index else None if graph is None or len(names) != 2: return None paths = [{u["source_path"] for u in index["units"] if u.get("person") == name} for name in names] if not all(paths): return None shared, left, right = compare_people(graph, paths[0], paths[1]) except Exception: return None lines = ["**Porovnanie** (podľa pojmov doložených v záznamoch):"] if shared: lines.append("- Spoločné: " + ", ".join(shared) + ".") if left: lines.append(f"- Len {names[0]}: " + ", ".join(left) + ".") if right: lines.append(f"- Len {names[1]}: " + ", ".join(right) + ".") return "\n".join(lines) if len(lines) > 1 else None def with_comparison(question: str, head: list[str], items: list[dict]) -> list[str]: low = normalized(question) # "Porovnaj A a B", "Mohol by A spolupracovať s B?" compare two people; "Kto by mohol spolupracovať # s A?" lists candidates (two of them are no pair to compare). if COMPARE.search(low) and len(items) == 2 and not TOPIC_LIST.match(low): block = comparison_block([items[0]["name"], items[1]["name"]]) if block: return [*head, block] return head def wiki_units(person: str) -> list[dict]: try: return [u for u in (_index() or {}).get("units", []) if u.get("person") == person] except Exception: return [] def wiki_works(person: str) -> dict[str, dict]: """work label -> {"type", "topic"} of the person's ZP Wiki works.""" works = {} for unit in wiki_units(person): label = unit.get("work_label") if not label: continue work = works.setdefault(label, {"type": unit.get("work_type"), "topic": None}) if unit.get("kind") == "topic" and not work["topic"] and not unit["text"].strip().endswith(":"): work["topic"] = re.sub(r"^\s*(?:Téma|Názov)\s*:\s*", "", unit["text"]).strip() return works def same_person_work(person: str, row: dict) -> bool: """A TUKE CRZP work of the person's name is confirmed as that person's when its title matches a ZP Wiki work topic or its supervisor is the person's ZP Wiki supervisor; else it may be a namesake's.""" def stems(text): return {w[:5] for w in normalized(text or "").split() if len(w) > 3} title = stems(row.get("title")) for work in wiki_works(person).values(): topic = stems(work.get("topic")) if title and topic and len(title & topic) >= max(1, min(len(title), len(topic)) // 2): return True from tools.knowledge.sources import profile_authors units = wiki_units(person) paths = {u["source_path"] for u in units} known = {normalized(profile_authors().get(p) or "") for p in paths} known |= {normalized(u["text"]) for u in units if u.get("kind") == "supervisor"} names = [set(n.split()) for n in known if n] # Both name parts must agree ("Hládek, Daniel" and the profile author "Daniel Hladek"). return any(len(set(normalized(s).replace(",", " ").split()) & n) >= 2 for s in row.get("supervisors", []) for n in names) FRAMEWORKS = {"python", "fastapi", "flask", "django", "streamlit", "react", "docker", "kubernetes", "git", "github", "gitlab", "pandas", "numpy", "jupyter", "colab", "linux", "javascript", "typescript", "java", "html", "css", "llm", "ai", "nlp", "veľký jazykový model", "umelá inteligencia", "zpwiki", "openwebui", "skills", "skill"} def wiki_terms(person: str) -> list[str]: """The person's most specific topics and tools (reviewed topic groups first, frameworks never): the words to look for in CRZP to find similar works.""" from tools.answers.content_selection import named_tools from tools.retrieval.synonyms import base_name, group_of counts: dict[str, float] = {} for unit in wiki_units(person): if unit.get("phase") not in {"result", "progress", "assignment"} and unit.get("kind") != "topic": continue for concept in unit.get("concepts") or []: if not isinstance(concept, str): continue if len(group_of(concept)) > 1: name, weight = base_name(concept), 2.0 elif named_tools(concept) and len(concept.split()) <= 2 and not re.search(r"[./]", concept): name, weight = concept, 1.0 else: continue if normalized(name) not in FRAMEWORKS: counts[name] = counts.get(name, 0) + weight return sorted(counts, key=lambda n: (-counts[n], n))[:4] def wiki_profile(person: str) -> str: """Topic titles and terms of the person's works: the text similar CRZP works are ranked against.""" works = wiki_works(person) return ". ".join([*(w["topic"] for w in works.values() if w.get("topic")), ", ".join(wiki_terms(person))]) # ------------------------------------------------------------------ CRZP part def crzp_plan(question: str, person: str | None, complete=None, student: bool = True): """(mode, CrzpQuery) or None when CRZP adds nothing to this question.""" from tools.crzp.answering.understand import Person, understand from tools.crzp.answering.units import unit_in_question low = normalized(question) explicit = bool(EXPLICIT.search(low)) defended_scope = bool(DEFENDED.search(low) and not person and ( unit_in_question(question) or re.search(r"\b(?:univerzit\w*|fakult\w*|katedr\w*|skol\w*|tuke|stu|kemt|fei)\b", low))) if WIKI_ONLY.search(low) and not explicit and not defended_scope: return None # A person ZP Wiki knows needs no model reading of the question (the person and school are # fixed below; rules give the intent): one model call less, the wiki part has the slots. query = understand(question, complete=complete, use_model=not person or explicit) if query.intent == "refused": return None other_school = query.school not in (None, "tuke") or explicit or bool(query.exclude_school) if person and other_school and not query.from_model: query = understand(question, complete=complete) if person and not other_school: if SIMILAR.search(low) and student: return "similar", query role = query.people[0].role if query.people else "any" if student: role = "author" if role == "author" else "any" # "Kto vedie Jána Ptáka?": his own works elif role not in ("supervisor", "opponent"): role = "supervisor" # a supervisor's works are those he or she led query.people = [Person(name=person, role=role, typed=person)] query.school, query.unit, query.from_model, query.title = "tuke", None, True, None query.topics, query.alternatives = [], [] return "person", query if query.people and not person and query.search_text() and ( THESIS.search(low) or PERSON_Q.search(low) and HUMAN_NAME.fullmatch(query.people[0].name)): return "explicit", query # a person ZP Wiki does not know: CRZP knows the theses of all schools if query.topics and not query.people and RELATED.search(low) and not re.search(r"\b(?:tag\w*|klucov\w*)\b", low): query.intent = "keywords" # "Ktoré témy súvisia s X?": topics of works about X, and who did them return "related", query if query.topics and not query.people and RELATED.search(low): query.intent = "keywords" # "Ktoré tagy súvisia s X?": the CRZP tags themselves return "explicit", query if query.topics and not query.people and query.intent in ("relation", "count", "keywords", "schools"): return "explicit", query # "Kto viedol najviac prác o X?", "Koľko prác o X…": computed over CRZP if (other_school or ((query.school or query.unit) and not query.people)) and query.search_text(): return "explicit", query # another school, CRZP itself, opponents, or a school/department as a whole if query.topics and not query.people and (TOPIC_LIST.search(low) or SIMILAR.search(low)): return "topic", query return None def clean_crzp(text: str) -> str: """CRZP answer text inside a combined answer: its own source lines go to the common "Zdroje CRZP" line.""" lines = [l for l in text.split("\n") if not re.match(r"^\s*(?:-\s*)?Zdroj:\s", l)] return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() def explicit_part(question: str, result: dict) -> dict | None: """A whole CRZP answer (CRZP is the main source), with its conversation token for follow-ups.""" from tools.crzp.answering import context as crzp_context if result.get("status") not in ("ok", "ambiguous"): return None rows = result.pop("_shown", None) or [] asked = result.pop("_asked", None) options = result.pop("_options", None) token = None if asked is not None: issued = crzp_context.issue(question, asked, rows, options) token = issued["token"] if issued else None entities = result.pop("_entities", None) or [] return {"mode": "explicit", "status": result["status"], "text": clean_crzp(result["answer"]), "rows": rows, "urls": result.get("source_urls", []), "crzp_token": token, "entities": entities} def _kind(row) -> str: from tools.crzp.answering.answer import _kind as kind return kind(row) def _names(values) -> str: from tools.crzp.source.parse import display_name return ", ".join(display_name(v) for v in values) or "neuvedený" def _work(row) -> str: return f"{_kind(row)} {row.get('year') or ''}".strip() + f" „{row['title']}“" def crzp_methods(person: str, row: dict, detail: dict | None) -> str: """"Ján Pták vo svojej bakalárskej práci „…“ (2025) využíva: ESP32, MQTT, InfluxDB, Home Assistant." — the work's keywords and the tools its abstract names.""" from tools.answers.content_selection import named_tools record = detail or row abstract = record.get("abstract") or "" found = [] for keyword in record.get("keywords", []): spelled = re.search(r"(?i)(? 1 and not any(f" {key} " in f" {normalized(t)} " for t in tools): tools = [t for t in tools if f" {normalized(t)} " not in f" {key} "] tools.append(name) where = {"bachelor_thesis": "vo svojej bakalárskej práci", "diploma_thesis": "vo svojej diplomovej práci", "dissertation": "vo svojej dizertačnej práci"}.get(row.get("work_type"), "vo svojej práci") return f"{person} {where} „{row['title']}“ ({row.get('year')}) využíva: {', '.join(tools[:12])}." def remember(rows: list[dict]) -> None: """Project the used CRZP works into Neo4j, linked to ZP Wiki people and topics (in the background).""" from tools.crzp.storage import graph rows = [r for r in rows if r.get("sid")] if rows and os.getenv("CRZP_GRAPH_LINKS", "1") != "0": _pool().submit(graph.project, rows) def crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None: """The CRZP contribution: {"mode", "text", "rows", "people", "works"} or None.""" part = _crzp_part(question, person, complete=complete, student=student) if part: remember(part.get("rows") or []) return part def _crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None: from tools.crzp.answering import answer as crzp from tools.crzp.source.client import CrzpUnavailable try: planned = crzp_plan(question, person, complete, student) if planned is None: return None mode, query = planned low = normalized(question) if mode == "similar": return crzp_similar(person, complete=complete) if mode == "related": return related_part(query) if mode == "explicit": result = crzp._answer_query(question, query, complete) part = explicit_part(question, result) if query.exclude_school == "tuke": # "na iných školách": the ZP Wiki (TUKE) part is not asked if part is None: # nothing found: CRZP's own "not found" answer, not the wiki's part = {"mode": "explicit", "status": "ok", "rows": [], "urls": [], "crzp_token": None, "entities": [], "text": clean_crzp(result.get("answer") or "V CRZP sa taká práca nenašla.")} part["home_excluded"] = True return part found, rows = crzp.find(query) if not rows: return None if mode == "topic": rows = relevant(question, query, rows)[:12] if not rows: return None # nothing in CRZP is really about the topic: no filler works return {"mode": mode, "items": [crzp_item(r) for r in rows], "rows": rows, "text": "", "urls": [r["url"] for r in rows if r.get("url")]} rows = crzp.ranked(question, query, rows) # person: that person's works at TUKE rows = [r for r in rows if crzp.university(r) == TUKE] if not rows: return None if not student: # a supervisor: the works he or she led, newest first rows = sorted(rows, key=lambda r: -(r.get("year") or 0)) lines = [f"- {_names(r.get('authors', []))} — „{r['title']}“ ({_kind(r)}, {r.get('year')})" for r in rows[:8]] more = f"\n\nSpolu {len(rows)} prác v CRZP na TUKE; zobrazených je 8 najnovších." if len(rows) > 8 else "" return {"mode": mode, "text": f"Práce, ktoré {person} viedol alebo oponoval, v CRZP (TUKE):\n\n" + "\n".join(lines) + more, "rows": rows[:8], "urls": [r["url"] for r in rows[:8] if r.get("url")]} lines, details = [], {} for row in rows[:3]: if ABOUT.search(low) or METHODS.search(low): details[row.get("sid")] = crzp.fetch.detail(row["sid"]) if row.get("sid") else None detail = details.get(row.get("sid")) if METHODS.search(low): lines.append(crzp_methods(person, row, detail)) elif ABOUT.search(low) and detail: label, text = crzp.summary(detail, complete) lines.append(f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{text}") else: authored = crzp.people_in_role(row, person, "author") lines.append(f"- {_work(row)} — " + ("" if authored else f"autor {_names(row.get('authors', []))}; ") + f"vedúci {_names(row.get('supervisors', []))}; " f"oponent {_names(row.get('opponents', []))}" + ("" if not authored or same_person_work(person, row) else " (v ZP Wiki k tejto práci záznam nie je; môže ísť o menovca)")) return {"mode": mode, "text": "\n\n".join(lines), "rows": rows[:3], "urls": [r["url"] for r in rows[:3] if r.get("url")], "works": [{"source": "crzp", "label": _work(r), "type": r.get("work_type"), "key": crzp.work_key(r)} for r in rows[:3]]} except CrzpUnavailable: return None def crzp_similar(person: str, *, complete=None) -> dict | None: """CRZP works sharing the person's specific tools/topics (searched in CRZP by them).""" from tools.crzp import fetch from tools.crzp.answering import answer as crzp from tools.crzp.source.parse import fold terms = wiki_terms(person) if not terms: return None scored: dict = {} for term in terms: for row in fetch.search(term)["rows"]: if any(fold(person.split()[-1]) in fold(a) for a in row.get("authors", [])): continue text = fold(row["title"] + " " + " ".join(row.get("keywords", []))) shared = [t for t in terms if fold(t) in text] if shared: key = crzp.work_key(row) scored[key] = (row, sorted(set(shared) | set(scored.get(key, (None, []))[1]))) if not scored: return None candidates = list(scored.values()) from tools.retrieval import rerank profile = wiki_profile(person) order = rerank.scores(profile, [r["title"] + ". " + ", ".join(r.get("keywords", [])) for r, _ in candidates]) \ or [0] * len(candidates) # Closest by content first; more shared topics/tools break ties. best = [c for _, c in sorted(zip(order, candidates), key=lambda i: (-i[0], -len(i[1][1])))][:12] return {"mode": "similar", "items": [crzp_item(r, shared) for r, shared in best], "text": "", "rows": [r for r, _ in best], "urls": [r["url"] for r, _ in best if r.get("url")]} _tag_cache: dict = {} _tag_lock = threading.Lock() def tag_frequencies() -> tuple[dict, int]: """In how many cached CRZP works each tag (its reviewed base name) occurs, and of how many works: a tag most works carry ("umelá inteligencia", "python") names no specific related topic.""" from collections import Counter from tools.crzp.fetch import store stamp = int(time.time() // 1800) # recounted at most every half hour with _tag_lock: if _tag_cache.get("stamp") == stamp: return _tag_cache["df"], _tag_cache["total"] from tools.retrieval.synonyms import base_name counts, seen = Counter(), set() for row in store().cached_rows(): key = (row.get("title"), row.get("year")) if key in seen: continue seen.add(key) counts.update({normalized(base_name(k.strip())) for k in row.get("keywords", []) if k.strip()}) with _tag_lock: _tag_cache.update(stamp=stamp, df=dict(counts), total=max(1, len(seen))) return _tag_cache["df"], _tag_cache["total"] def wiki_people_for(topic: str) -> list[str]: """ZP Wiki students whose own records (topic, assignment, progress, results) name the topic.""" from tools.retrieval.synonyms import variants_pattern pattern = variants_pattern(topic) people = {} for unit in (_index() or {}).get("units", []): if unit.get("kind") in {"reference", "meeting", "supervisor", "consultant", "lifecycle"}: continue if pattern.search(normalized(unit.get("text", ""))) or any( pattern.fullmatch(normalized(c)) for c in unit.get("concepts", []) if isinstance(c, str)): people.setdefault(unit["person"], unit.get("source_url")) return list(people) def related_part(query) -> dict | None: """Topics related to the asked one: the CRZP keywords of works about it, weighted by how specific they are, each with the ZP Wiki students and CRZP works that have it.""" import math from tools.crzp.answering import answer as crzp from tools.retrieval.synonyms import base_name, group_of, variants_pattern found, rows = crzp.find(query) if not rows: return None asked = [t for t in [*query.topics, *query.phrases] if t] patterns = [variants_pattern(t) for t in asked] asked_words = {w for t in asked for x in group_of(t) for w in normalized(x).split()} generic = {normalized(g) for g in crzp.GENERIC_TAGS} | {"umela inteligencia", "strojove ucenie", "python"} df, total = tag_frequencies() stats: dict = {} for row in rows: for keyword in {base_name(k.strip()) for k in row.get("keywords", []) if k.strip()}: key = normalized(keyword) if (len(key) < 3 or key in generic or set(key.split()) <= asked_words or any(p.fullmatch(key) for p in patterns)): continue # the asked topic itself, a synonym, a part of it or a generic word entry = stats.setdefault(key, {"label": keyword, "rows": []}) if keyword[:1].isupper() and not entry["label"][:1].isupper(): entry["label"] = keyword entry["rows"].append(row) ranked = sorted((e for e in stats.values() if len(e["rows"]) >= 2), key=lambda e: -len(e["rows"]) * math.log(total / (1 + df.get(normalized(e["label"]), 0)))) # The same topic in two languages ("Konverzačný marketing" / "Conversational marketing") is one # topic: CRZP works carry bilingual keywords, so both tag (nearly) the same works. from difflib import SequenceMatcher merged = [] for entry in ranked: works = {crzp.work_key(r) for r in entry["rows"]} key = normalized(entry["label"]) twin = next((m for m in merged if len(works & m["works"]) >= 0.8 * max(len(works), len(m["works"])) and SequenceMatcher(None, key, normalized(m["label"])).ratio() >= 0.6), None) if twin is None: merged.append({**entry, "works": works}) ranked = merged[:18] if not ranked: return None items = [] for entry in ranked: label = entry["label"][:1].upper() + entry["label"][1:] example = max(entry["rows"], key=lambda r: r.get("year") or 0) authors = example.get("authors") or [] from tools.crzp.source.parse import display_name wiki = wiki_people_for(entry["label"]) from tools.answers.unified import catalog as wiki_catalog, join text = (f"{crzp.plural(len(entry['rows']))}, napr. „{example['title']}“ " f"({display_name(authors[0]) if authors else 'autor neuvedený'}, {crzp.university(example)}, " f"{example.get('year') or 'rok neuvedený'})") if wiki: text += "; venujú sa mu aj " + join(wiki[:4]) + (f" a ďalší ({len(wiki) - 4})" if len(wiki) > 4 else "") people = wiki_catalog()["people"] urls = [example["url"]] if example.get("url") else [] urls += [(people.get(name) or {}).get("url") for name in wiki[:4]] items.append({"name": label, "source": "topic", "text": text, "urls": [u for u in urls if u]}) topic = base_name(asked[0]) if asked else "" return {"mode": "related", "items": items, "topic": topic, "rows": [e["rows"][0] for e in ranked], "text": "", "urls": [i["urls"][0] for i in items if i["urls"]]} def crzp_item(row: dict, shared: list[str] | None = None) -> dict: """One CRZP work as an item of a combined list.""" from tools.crzp.answering import answer as crzp from tools.crzp.source.parse import display_name authors = row.get("authors") or [] name = display_name(authors[0]) if authors else "autor neuvedený" school = crzp.university(row) line = f"{school}: „{row['title']}“ ({_kind(row)}, {row.get('year') or 'rok neuvedený'})" if shared: line += "; spoločné: " + ", ".join(shared) return {"name": name, "school": school, "source": "crzp", "text": line, "year": row.get("year"), "urls": [row["url"]] if row.get("url") else [], "sid": row.get("sid"), "title": row["title"]} def relevant(question: str, query, rows: list[dict]) -> list[dict]: """CRZP works really about the asked topic, most relevant first: the topic (or a reviewed synonym, in any case form) in the title or the CRZP keywords, or a cross-encoder score above CRZP_TOPIC_MIN_SCORE. A work found only through its abstract ("Fairy Tale Intertexts" for "anotácie") is no answer to a topic question.""" from tools.retrieval import rerank from tools.retrieval.synonyms import variants_pattern if not rows: return [] terms = [t for t in [*query.topics, *query.phrases] if t] patterns = [variants_pattern(t) for t in terms] texts = [r["title"] + ". " + ", ".join(r.get("keywords", [])) for r in rows] asked = " ".join([question, *[t for t in [*query.topics, *query.alternatives] if normalized(t) not in normalized(question)]]) scores = rerank.scores(asked, texts) if scores is not None: # the cross-encoder scores at most MAX_PAIRS texts; the rest only by name scores = list(scores) + [float("-inf")] * (len(texts) - len(scores)) kept = [] for i, (row, text) in enumerate(zip(rows, texts)): named = any(p.search(normalized(text)) for p in patterns) score = scores[i] if scores is not None else (1.0 if named else -1.0) if named or score >= TOPIC_MIN_SCORE: kept.append((score + (1.0 if named else 0.0), i, row)) return [row for _, _, row in sorted(kept, key=lambda k: (-k[0], k[1]))] # ------------------------------------------------------------------ items of a list answer HEADING = re.compile(r"^\*\*([^*\n]+?)\*\*\s*$") PERSON_NAME = re.compile(r"[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][\w'’.-]+(?:\s+[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][\w'’.-]+){1,3}") # Blocks after the people of a wiki list (they belong to the whole list). TAIL = re.compile(r"^(?:Téma je len v zadaní|Len štúdium alebo poznámky|Tému majú len|Podklady k porovnávanej osobe)") SOURCE_LINE = re.compile(r"^Zdroj(?:e)?(?: CRZP)?:\s*(.+)$", re.M) def split_sources(text: str) -> tuple[str, list[str]]: """The answer without its source lines, and their links.""" urls = [] for match in SOURCE_LINE.finditer(text or ""): urls += re.findall(r"https?://[^\s,)\]]+", match.group(1)) body = SOURCE_LINE.sub("", text or "") return re.sub(r"\n{3,}", "\n\n", body).strip(), list(dict.fromkeys(urls)) def _slug(name: str) -> str: return "_".join(normalized(name).split()) def wiki_items(body: str, urls: list[str]) -> tuple[list[str], list[dict], list[str], list[str]]: """(head blocks, one item per person, tail blocks, links not tied to a person) of a wiki answer.""" blocks = [b.strip() for b in re.split(r"\n\s*\n", body) if b.strip()] head, items, tail = [], [], [] current = None for block in blocks: first = block.split("\n", 1)[0] heading = HEADING.match(first) name = re.split(r"\s+—\s+", heading.group(1))[0].strip() if heading else "" if tail or TAIL.match(block): tail.append(block) elif heading and PERSON_NAME.fullmatch(name) and not name.endswith(":"): same = next((i for i in items if i["name"] == name), None) if same is not None: current = same current["blocks"].append(block) # the same person's next work (also when listed apart) else: current = {"name": name, "school": WIKI_SCHOOL, "source": "wiki", "blocks": [block]} items.append(current) elif current is None: head.append(block) else: current["blocks"].append(block) general = [] for url in urls: owner = next((i for i in items if _slug(i["name"]) in url), None) if owner is not None: owner.setdefault("urls", []).append(url) else: general.append(url) for item in items: item["text"] = "\n\n".join(item.pop("blocks")) item.setdefault("urls", []) return head, items, tail, general def render_item(number: int, item: dict, mixed: bool) -> str: if item["source"] in ("topic", "crzp", "line"): return f"**{number}. {item['name']}** — {item['text']}" first, _, rest = item["text"].partition("\n") heading = HEADING.match(first) title = heading.group(1) if heading else item["name"] return f"**{number}. {title}**" + ("\n" + rest if rest else "") def merge_people(wiki: list[dict], crzp_items: list[dict]) -> list[dict]: """A CRZP work of a person already in the ZP Wiki list joins that person's item (no person twice).""" by_name = {normalized(item["name"]): item for item in wiki} rest = [] for item in crzp_items: owner = by_name.get(normalized(item["name"])) if owner is not None and item.get("school") == TUKE: owner["text"] += f"\n- V CRZP: {item['text'].split(': ', 1)[-1]}" owner.setdefault("urls", []).extend(item.get("urls", [])) else: rest.append(item) return rest def newest_first(items: list[dict]) -> list[dict]: """People of a list from the one with the newest work or record (stable for equal dates).""" from tools.answers.unified import recency return sorted(items, key=lambda i: recency(i["name"]) if i.get("source") == "wiki" else str(i.get("year") or ""), reverse=True) def first_page(wiki: list[dict], crzp_items: list[dict]) -> tuple[list[dict], list[dict]]: """(shown, rest): ZP Wiki people first (at most WIKI_FIRST when CRZP has works too), CRZP fills the page; each group from the newest work.""" wiki = newest_first(wiki) crzp_items = newest_first(merge_people(wiki, crzp_items)) take_wiki = min(len(wiki), WIKI_FIRST if crzp_items else PAGE) take_crzp = min(len(crzp_items), PAGE - take_wiki) take_wiki = min(len(wiki), PAGE - take_crzp) shown = wiki[:take_wiki] + crzp_items[:take_crzp] return shown, wiki[take_wiki:] + crzp_items[take_crzp:] def more_line(rest: list[dict]) -> str: if all(i["source"] == "topic" for i in rest): word = "ďalšia téma" if len(rest) == 1 else "ďalšie témy" if len(rest) < 5 else "ďalších tém" return f"Našl{'a' if len(rest) == 1 else 'i' if len(rest) < 5 else 'o'} sa ešte {len(rest)} {word}. Chceš vidieť ďalšie?" word = "ďalšia osoba" if len(rest) == 1 else "ďalšie osoby" if len(rest) < 5 else "ďalších osôb" return f"Našl{'a' if len(rest) == 1 else 'i' if len(rest) < 5 else 'o'} sa ešte {len(rest)} {word}. Chceš vidieť ďalšie?" def compose_list(head: list[str], shown: list[dict], rest: list[dict], tail: list[str], general_urls: list[str], *, start: int = 1, mixed: bool) -> tuple[str, list[str], str | None]: """(text, sources, id of the stored rest).""" blocks = list(head) + [render_item(start + i, item, mixed) for i, item in enumerate(shown)] + list(tail) urls = list(dict.fromkeys([*general_urls, *[u for item in shown for u in item.get("urls", [])]])) if urls: blocks.append("Zdroje: " + ", ".join(urls)) from tools.answers import pages page_id = pages.put(rest, {"mixed": mixed}) if rest else None if rest and page_id: blocks.append(more_line(rest)) return "\n\n".join(blocks), urls, page_id def entity(item: dict, number: int) -> dict: return {k: v for k, v in {"n": number, "name": item["name"], "school": item.get("school"), "source": item.get("source"), "sid": item.get("sid"), "title": item.get("title")}.items() if v is not None} # ------------------------------------------------------------------ composition def _ok(question, engine, text, urls, base) -> dict: keep = {k: v for k, v in base.items() if k not in ("answer", "status", "reason", "delivery_check", "source_urls", "conversation_context")} return {**keep, "status": "ok", "reason": None, "accepted_for_delivery": True, "answer": text, "source_urls": list(dict.fromkeys(urls)), "claims": base.get("claims", []), "index_version": base.get("index_version"), "complete": base.get("complete", True), "truncated": base.get("truncated", False), "truncation_reasons": base.get("truncation_reasons", []), "delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": fingerprint(question, engine)}} def _with_token(result: dict, *, wiki_context=None, entities=(), options=(), question="", crzp=None, primary="wiki", page=None) -> dict: wiki_context = wiki_context or {} token = seal({"wiki": wiki_context.get("token"), "entities": list(entities)[:40], "options": list(options)[:6], "question": question, "crzp": crzp, "primary": primary, "page": page}) if token: result["conversation_context"] = {**{k: v for k, v in wiki_context.items() if k != "token"}, "token": token, "work_options": wiki_context.get("work_options", []), "meeting_options": wiki_context.get("meeting_options", [])} return result NEXT_PAGE = re.compile(r"\b(?:dalsi\w*|dalej|viac|ostatn\w*|zvysn\w*|pokracuj\w*|zvysok)\b") # Words of a paging reply ("áno, ukáž ďalších", "pokračuj"): a message with any other word is a new question # ("a čo robil ďalej?", "povedz viac o jeho práci"). PAGER_WORDS = {"a", "ano", "aj", "jasne", "ok", "okej", "hej", "chcem", "daj", "ukaz", "zobraz", "vypis", "prosim", "urcite", "samozrejme", "mi", "ich", "ho", "este", "dalsich", "dalsie", "dalsi", "dalej", "viac", "ostatnych", "ostatne", "zvysnych", "zvysne", "zvysok", "pokracuj", "pokracovat", "vidiet", "chcel", "chcela", "by", "som", "tych", "ty", "tie", "osoby", "ludi", "studentov", "temy", "prace", "polozky", "stranu", "dalsiu", "zoznam"} # A question starting like a new one is never a choice reply ("Kto robil RAG?" while a work choice is pending). NEW_QUESTION = re.compile(r"^\W*(?:kto|co|ako|kedy|kde|ktor\w*|kolk\w*|ak[eyaou]\w*|preco|najdi|porovnaj|vypis|ukaz)\b") YES = re.compile(r"^\W*(?:ano|jasne|ok|okej|hej|chcem|daj|ukaz|prosim|urcite|samozrejme)(?:\s+(?:ano|prosim|daj|ukaz|ich|mi|ho|aj))*\W*$") ORDINAL_WORDS = {"prv": 1, "druh": 2, "tret": 3, "stvrt": 4, "piat": 5, "siest": 6, "siedm": 7, "osm": 8, "deviat": 9, "desiat": 10} REFERENCE = re.compile( r"(?i)\b(?:(?:t[eé]n|t[aá]|t[uú]|toho|tej|tomu|tom|tou)\s+)?" r"(prv|druh|tret|štvrt|stvrt|piat|šiest|siest|siedm|ôsm|osm|deviat|desiat|posledn)\w*" r"(?:\s+(?:z\s+nich|zo\s+zoznamu|v\s+poradí|v\s+poradi|študent\w*|student\w*|osob\w*|človek\w*|clovek\w*))" r"|\b(?:číslo|cislo|č\.)\s*(\d{1,2})\b") def _reference(question: str, entities: list[dict]) -> tuple[re.Match, dict] | None: """"prvý z nich", "toho tretieho študenta", "číslo 2" -> the shown person it points at.""" match = REFERENCE.search(question) if not match or not entities: return None if match.group(2): number = int(match.group(2)) else: word = normalized(match.group(1)) number = len(entities) if word.startswith("posledn") else next( (n for stem, n in ORDINAL_WORDS.items() if word.startswith(stem)), None) chosen = next((e for e in entities if e.get("n") == number), None) return (match, chosen) if chosen else None def _stamp(result: dict, question: str, engine: str) -> dict: """An answer prepared for a question this layer rewrote ("prvý z nich" -> the name, "2" -> the pending question about the chosen work) is bound to the message the user sent. Any other fingerprint stays as it is, so the client still rejects an answer that belongs elsewhere.""" internal = result.pop("_asked_as", None) or [] result.pop("_plan", None) # internal summary of the wiki plan check = result.get("delivery_check") if isinstance(check, dict) and check.get("request_fingerprint") in {fingerprint(q, engine) for q in internal}: result["delivery_check"] = {**check, "request_fingerprint": fingerprint(question, engine)} return result _warmed = threading.Event() def answer(question: str, *, engine: str, limit: int = 5, context_token: str | None = None, complete=None, deliver=None) -> dict: """deliver: the ZP Wiki answer function (the API passes its own, tests replace it).""" question = question.strip() if enabled() and not _warmed.is_set(): # tag rarity for related topics, counted once in the background _warmed.set() _pool().submit(tag_frequencies) return _stamp(_answer(question, engine=engine, limit=limit, context_token=context_token, complete=complete, deliver=deliver), question, engine) def _ask_choice(question: str, engine: str, previous: dict) -> dict: options = previous["options"] text = ("Odpovedz prosím číslom: " + ", ".join(f"{i} – {o['label']}" for i, o in enumerate(options, 1)) + ".") result = _ok(question, engine, text, [], {}) result.update(status="ambiguous", reason="clarification_required", delivery_check={"layer": "planning", "code": "clarification_required", "request_fingerprint": fingerprint(question, engine)}) return _with_token(result, options=options, question=previous["question"]) def _next_page(question: str, engine: str, previous: dict) -> dict: from tools.answers import pages stored = pages.get(previous.get("page")) entities = previous.get("entities") or [] if not stored: return _ok(question, engine, "Ďalšie položky predchádzajúcej odpovede už nie sú k dispozícii; zopakuj otázku.", [], {}) items, mixed = stored["items"], stored["meta"].get("mixed", False) start = max((e.get("n", 0) for e in entities), default=0) + 1 text, urls, page_id = compose_list([], items[:PAGE], items[PAGE:], [], [], start=start, mixed=mixed) shown = entities + [entity(item, start + i) for i, item in enumerate(items[:PAGE])] return _with_token(_ok(question, engine, text, urls, {}), entities=shown, question=question, page=page_id, crzp=previous.get("crzp"), primary=previous.get("primary", "wiki")) def _schools_answer(question: str, engine: str, previous: dict) -> dict: entities = previous["entities"] lines = [f"{e.get('n', i)}. {e['name']} — {e.get('school') or 'škola neuvedená'}" for i, e in enumerate(entities, 1)] text = "Školy ľudí z predchádzajúcej odpovede:\n\n" + "\n".join(lines) return _with_token(_ok(question, engine, text, [], {}), entities=entities, question=question, page=previous.get("page"), crzp=previous.get("crzp"), primary=previous.get("primary", "wiki")) def _crzp_work_answer(question: str, engine: str, chosen: dict, complete=None) -> dict: """A follow-up about a CRZP work of the previous list ("o čom je práca tretieho z nich?").""" from tools.crzp.answering import answer as crzp from tools.crzp.answering.understand import Person, understand query = understand(question, complete=complete) query.people = [Person(name=chosen["name"], role="author", typed=chosen["name"])] query.sids, query.title, query.topics, query.alternatives = [chosen["sid"]], None, [], [] query.school = query.unit = None query.from_model = True if query.intent in ("list", "similar", "schools"): query.intent = "about" result = crzp._answer_query(question, query, complete) for key in ("_shown", "_asked", "_options", "_entities"): result.pop(key, None) if result.get("accepted_for_delivery") is not True: return result from tools.answers.unified import strip_sources body, urls = split_sources(strip_sources(clean_crzp(result["answer"]))) urls = list(dict.fromkeys([*urls, *(result.get("source_urls") or [])])) text = body + ("\n\nZdroje: " + ", ".join(urls) if urls else "") return {**result, "answer": text, "source_urls": urls, "delivery_check": {**result["delivery_check"], "request_fingerprint": fingerprint(question, engine)}} def _answer(question: str, *, engine: str, limit: int, context_token: str | None, complete, deliver) -> dict: from tools.answers.answer_delivery import deliver_answer as default_deliver deliver_answer = deliver or default_deliver from tools.crzp.answering.wiki_bridge import wiki_match, wiki_person previous = unseal(context_token) if previous and previous.get("error"): from tools.answers.answer_errors import public_failure failure = public_failure({"reason": previous["error"], "engine": engine}) failure["delivery_check"] = {"layer": "context", "code": "unavailable", "request_fingerprint": fingerprint(question, engine)} return failure wiki_token = previous.get("wiki") if previous else context_token low = normalized(question) if previous and previous.get("options"): # "Ktorú prácu myslíš?" was asked chosen = _choice(low, previous["options"]) if chosen is not None: result = _answer_choice(previous["question"], chosen, engine=engine, limit=limit, complete=complete, deliver=deliver_answer) result["_asked_as"] = [previous["question"], *result.pop("_asked_as", [])] if result.get("accepted_for_delivery") is True and result.get("status") == "ok": # the offer stays: "a tú druhú?" / "1" after "2" answers the same question for the other work result = _with_token(result, wiki_context=result.get("conversation_context"), options=previous["options"], question=previous["question"]) return result if len(low.split()) <= 3 and not wiki_person(question) and not NEW_QUESTION.match(low): return _ask_choice(question, engine, previous) words = re.findall(r"[a-z0-9]+", low) wants_more = bool(YES.match(low) or NEXT_PAGE.search(low) and words and all(w in PAGER_WORDS for w in words)) if previous and previous.get("page") and wants_more: return _next_page(question, engine, previous) if previous and previous.get("entities") and wants_more: # nothing more was offered return _with_token(_ok(question, engine, "Všetko nájdené už je v predchádzajúcej odpovedi.", [], {}), entities=previous["entities"], question=question, crzp=previous.get("crzp"), primary=previous.get("primary", "wiki")) entities = (previous or {}).get("entities") or [] # "a z akých škôl sú?" asks about the people shown; "Ktoré školy majú práce o RAG?" is a new question. if entities and SCHOOLS.search(low) and not wiki_person(question) and not THESIS.search(low) and len(words) <= 8: return _schools_answer(question, engine, previous) asked = question full = ellipsis(question, previous) or again(question, previous) if full: # "a Ján Pták?": the previous question about this person; "ale čo robil?": that question again return {**_answer(full, engine=engine, limit=limit, context_token=None, complete=complete, deliver=deliver), "_asked_as": [full]} reference = _reference(question, entities) if entities else None if reference: match, chosen = reference if chosen.get("source") == "crzp" and chosen.get("sid"): return _crzp_work_answer(question, engine, chosen, complete) question = (question[:match.start()] + chosen["name"] + question[match.end():]).strip() low, wiki_token, previous = normalized(question), None, None # a question about that person now return {**_answer(question, engine=engine, limit=limit, context_token=None, complete=complete, deliver=deliver), "_asked_as": [question]} match = wiki_match(question) from tools.crzp.answering.context import CONTINUES from tools.crzp.source.parse import fold continues = bool(CONTINUES.search(fold(question))) or len(words) <= 4 if previous and previous.get("primary") == "crzp" and previous.get("crzp") and not match and continues: # The last answer came from CRZP ("O čom je práca Jána Nováka?" -> "2", "a kto bol oponent?"). from tools.crzp.answering import answer as crzp followed = crzp._answer(question, complete, context_token=previous["crzp"]) followed.pop("_shown", None), followed.pop("_asked", None), followed.pop("_options", None) shown = followed.pop("_entities", None) or [] if followed.get("accepted_for_delivery") is True: from tools.answers.unified import strip_sources body, urls = split_sources(strip_sources(clean_crzp(followed["answer"]))) urls = list(dict.fromkeys([*urls, *(followed.get("source_urls") or [])]))[:12] result = {**followed, "answer": body + ("\n\nZdroje: " + ", ".join(urls) if urls else ""), "delivery_check": {**followed["delivery_check"], "request_fingerprint": fingerprint(question, engine)}} token = (followed.get("conversation_context") or {}).get("token") result.pop("conversation_context", None) entities = [{**e, "n": i} for i, e in enumerate(shown, 1)] or previous.get("entities", []) return _with_token(result, entities=entities, question=question, crzp=token or previous["crzp"], primary="crzp") person, student = (match["name"], match["student"]) if match else (None, True) started = time.monotonic() pool = _pool() from tools.answers import timing import contextvars def timed(name, function, *args, **kwargs): began = time.monotonic() try: return function(*args, **kwargs) finally: timing.step(name, time.monotonic() - began) wiki_future = pool.submit(contextvars.copy_context().run, timed, "wiki", deliver_answer, question, engine=engine, limit=limit, **({"context_token": wiki_token} if wiki_token else {})) crzp_future = pool.submit(contextvars.copy_context().run, timed, "crzp", crzp_part, question, person, complete=complete, student=student) if enabled() else None wiki = wiki_future.result() try: part = crzp_future.result(timeout=max(1.0, CRZP_SECONDS - (time.monotonic() - started))) if crzp_future else None except FutureTimeout: part = None # CRZP too slow this time: the wiki answer is not held back (its search stays cached) except Exception: part = None return compose(asked, question, engine, wiki, part, person) def compose(asked: str, question: str, engine: str, wiki: dict, part: dict | None, person: str | None) -> dict: """One answer from the ZP Wiki result and the CRZP part (without naming the sources in the text).""" from tools.answers.unified import strip_sources result = _compose(asked, question, engine, wiki, part, person) if result.get("accepted_for_delivery") is True and isinstance(result.get("answer"), str): result["answer"] = strip_sources(result["answer"]) return result def _unified(asked, question, engine, wiki, wiki_ok, part, person, wiki_context, plan) -> dict | None: """The answer of a question about one person, a supervisor's students or two people, built from the records of both sources (tools/answers/unified.py); None leaves the general composition.""" from tools.answers import unified as U if wiki.get("reason") in REFUSED or wiki.get("status") == "ambiguous" and person is None: return None low = normalized(question) rows = (part or {}).get("rows") or [] wiki_text = wiki.get("answer") if wiki_ok and isinstance(wiki.get("answer"), str) else "" base = wiki if wiki_ok else {"index_version": wiki.get("index_version")} def done(out, **token): text = out["text"] + ("\n\nZdroje: " + ", ".join(out["urls"]) if out.get("urls") else "") return _with_token(_ok(question, engine, text, out.get("urls") or [], base), wiki_context=wiki_context, question=question, **token) if person is None and wiki.get("resolved_query"): # "a aké úlohy dostal?": the person of the resolved question found = _one_person(wiki["resolved_query"]) person = found["name"] if found else None students = U.catalog()["people"] student = bool(person and (students.get(person) or {}).get("person_kind") in ("student", "intern")) # A supervisor: how many students, and who (newest work first, each student once). if wiki_ok and COUNT_STUDENTS.search(low): # also a supervisor who has a student profile of their own count = re.search(r"počet študentov[^:]*:\s*(\d+)", wiki_text) names = re.search(r"Študenti:\s*(.+?)\.\s*$", wiki_text, re.M) who = person or (re.match(r"^(.+?) — počet", wiki_text) or [None, None])[1] if count and names and who: listed = [U.spelled(n.strip()) for n in names.group(1).split(",")] return done({"text": f"{U.spelled(who)} vedie {U.plural_students(int(count.group(1)))}: {U.join(listed)}.", "urls": wiki.get("source_urls") or []}) if U.ROSTER_Q.search(low) and (wiki_ok or rows): who = person or "" body, urls = split_sources(wiki_text) head, items, tail, general = wiki_items(body, urls or wiki.get("source_urls") or []) roster = U.roster_items(who, items, rows) if who else [] if roster: shown, rest = roster[:PAGE], roster[PAGE:] heading = [f"{U.spelled(who)} vedie tieto práce (od najnovších):"] text, shown_urls, page_id = compose_list(heading, shown, rest, [], [], mixed=False) return _with_token(_ok(question, engine, text, shown_urls, base), wiki_context=wiki_context, question=question, page=page_id, entities=[entity(item, i) for i, item in enumerate(shown, 1)]) # "Kto by mohol spolupracovať s X?", "Nájdi práce podobné práci X": the candidates, newest work first if student and wiki_ok and (plan.get("intent") == "similar" or SIMILAR.search(low)) and not COMPARE_TWO(low): body, urls = split_sources(wiki_text) head, items, tail, general = wiki_items(body, urls or wiki.get("source_urls") or []) if items: candidates = U.similar_items(newest_first(items)) crzp_items = [i for i in (part or {}).get("items") or [] if i.get("source") == "crzp"] anchor = U.anchor_sentence(person) heading = [(anchor + " " if anchor else "") + "Podobnej téme sa venujú (od najnovších prác):"] shown, rest = first_page(candidates, crzp_items) text, shown_urls, page_id = compose_list(heading, shown, rest, [], [*general, *(U.work_urls(person, []))], mixed=True) return _with_token(_ok(question, engine, text, shown_urls, base), wiki_context=wiki_context, question=question, page=page_id, entities=[entity(item, i) for i, item in enumerate(shown, 1)]) if student: works = U.person_works(person, rows) out = None if STUDY_START.search(low) and not re.search(r"\b(?:prac\w*|projekt\w*|nazov|tem\w*|veduc\w*)\b", low): out = U.start_year(person) # only the start of studies; a compound question stays with the wiki answer elif plan.get("intent") == "meeting" or re.search(r"\bstretnut\w*", low) or U.TASKS_Q.search(low) and wiki_text: text = U.meeting(person, wiki_text, low, follow_up=asked != question or bool(re.match(r"^\W*a\b", normalized(asked)))) out = {"text": text, "urls": wiki.get("source_urls") or U.work_urls(person, [])} if text else None elif U.SUPERVISOR_Q.search(low): out = U.supervisors(person, works, low, wiki_text) elif U.METHODS_Q.search(low): options = U.thesis_options(person, works, low) if options: text = (f"{U.spelled(person)} má {len(options)} záverečné práce:\n\n" + "\n".join(f"{i}. {o['label']}" for i, o in enumerate(options, 1)) + "\n\nZ ktorej chceš vedieť, aké technológie využíva?") result = _ok(question, engine, text, [], base) result.update(status="ambiguous", reason="clarification_required", delivery_check={"layer": "planning", "code": "clarification_required", "request_fingerprint": fingerprint(question, engine)}) return _with_token(result, wiki_context=wiki_context, options=options, question=question) out = U.methods(person, works, wiki_text, (part or {}).get("text") or "", low) elif U.ABOUT_Q.search(low): out = U.about(person, works, low) elif U.WORKS_Q.search(low) or U.DID_Q.match(low) and not plan.get("period") and not plan.get("years"): out = U.works_list(person, works) if out: return done(out) # Two people compared: one table. if wiki_ok and COMPARE.search(low) and not TOPIC_LIST.match(low): body, urls = split_sources(wiki_text) head, items, tail, general = wiki_items(body, urls) if len(items) == 2: out = U.comparison([items[0]["name"], items[1]["name"]], collaboration=bool(re.search(r"\bspolupr", low))) if out: return done(out) return None def _compose(asked: str, question: str, engine: str, wiki: dict, part: dict | None, person: str | None) -> dict: """One answer from the ZP Wiki result and the CRZP part.""" from tools.answers.finalize import finalize low = normalized(question) if wiki.get("reason") in REFUSED or wiki.get("status") == "ambiguous" and ( person or wiki.get("reason") not in WIKI_CLARIFICATION_YIELDS): part = None # refused/out of scope, or the wiki asks first (which person, which work): no CRZP wiki_ok = wiki.get("accepted_for_delivery") is True and wiki.get("status") == "ok" if part is not None and part["mode"] == "explicit" and wiki.get("status") == "ambiguous": wiki_ok = False if wiki.get("accepted_for_delivery") is True and isinstance(wiki.get("answer"), str): wiki = {**wiki, "answer": finalize(wiki["answer"], wiki.get("source_urls"))} wiki_context = wiki.get("conversation_context") plan = wiki.pop("_plan", None) or {} special = _unified(asked, question, engine, wiki, wiki_ok, part, person, wiki_context, plan) if special is not None: return special if part is None or wiki.get("status") == "ambiguous" and part["mode"] != "explicit": # (a wiki clarification such as "Uveď meno študenta" yields to a question CRZP answers as a whole) if wiki.get("accepted_for_delivery") is not True: return wiki if wiki_ok: body, urls = split_sources(wiki["answer"]) head, items, tail, general = wiki_items(body, urls or wiki.get("source_urls") or []) if len(items) >= 2: head = with_comparison(question, head, items) shown, rest = first_page(items, []) text, shown_urls, page_id = compose_list(head, shown, rest, tail, general, mixed=False) result = {**wiki, "answer": text, "source_urls": shown_urls} return _with_token(result, wiki_context=wiki_context, question=question, page=page_id, entities=[entity(item, i) for i, item in enumerate(shown, 1)]) return _with_token(wiki, wiki_context=wiki_context, question=question) # Different works in the two sources and the question names none: ask which one. if person and part["mode"] == "person" and (METHODS.search(low) or ABOUT.search(low)): options = _work_options(person, part, low) if options: text = (f"{person} má v podkladoch viac prác. Ktorú myslíš?\n\n" + "\n".join(f"{i}. {o['label']} ({'ZP Wiki' if o['source'] == 'wiki' else 'CRZP'})" for i, o in enumerate(options, 1)) + "\n\nOdpovedz číslom alebo typom práce.") result = _ok(question, engine, text, [], wiki) result.update(status="ambiguous", reason="clarification_required", delivery_check={"layer": "planning", "code": "clarification_required", "request_fingerprint": fingerprint(question, engine)}) return _with_token(result, wiki_context=wiki_context, options=options, question=question) if part["mode"] == "explicit" and part.get("status") == "ambiguous": # CRZP asks which namesake result = _ok(question, engine, part["text"], [], wiki if wiki_ok else {"index_version": wiki.get("index_version")}) result.update(status="ambiguous", reason="clarification_required", delivery_check={"layer": "planning", "code": "clarification_required", "request_fingerprint": fingerprint(question, engine)}) return _with_token(result, question=question, crzp=part.get("crzp_token"), primary="crzp") if part.get("home_excluded"): wiki_ok = False base = wiki if wiki_ok else {"index_version": wiki.get("index_version")} wiki_body, wiki_urls = split_sources(wiki["answer"]) if wiki_ok else ("", []) wiki_urls = wiki_urls or (wiki.get("source_urls") or [] if wiki_ok else []) if part["mode"] == "related": # "Ktoré témy súvisia s X?": the topics and who did them head = [f"S témou „{part['topic']}“ súvisia tieto témy (vyskytujú sa v tých istých prácach):"] shown, rest = part["items"][:PAGE], part["items"][PAGE:] text, urls, page_id = compose_list(head, shown, rest, [], [], mixed=False) return _with_token(_ok(question, engine, text, urls, base), wiki_context=wiki_context, question=question, entities=[entity(item, i) for i, item in enumerate(shown, 1)], page=page_id) if part.get("items") is not None: # topic or similar works: one numbered list with the wiki people head, items, tail, general = wiki_items(wiki_body, wiki_urls) if wiki_ok else ([], [], [], []) if wiki_ok and not items: # a wiki answer that is no list stays whole, the CRZP works follow it head, tail, general = [wiki_body], [], wiki_urls if not wiki_ok: head = ["V ZP Wiki k tomu nie je záznam. Práce na túto tému v CRZP (Centrálny register záverečných prác):"] shown, rest = first_page(items, part["items"]) text, urls, page_id = compose_list(head, shown, rest, tail, general, mixed=True) return _with_token(_ok(question, engine, text, urls, base), wiki_context=wiki_context, question=question, entities=[entity(item, i) for i, item in enumerate(shown, 1)], page=page_id) crzp_body, crzp_urls = split_sources(part["text"]) crzp_urls = list(dict.fromkeys([*crzp_urls, *part.get("urls", [])]))[:12] entities = [{**e, "n": i} for i, e in enumerate(part.get("entities") or [], 1)] if wiki_ok: # a list of wiki people ("Koho vedie X?") is paged; the CRZP part follows the first page head, items, tail, general = wiki_items(wiki_body, wiki_urls) if len(items) >= 2: head = with_comparison(question, head, items) shown, rest = first_page(items, []) tail = [*tail, "V CRZP (Centrálny register záverečných prác):\n\n" + crzp_body] text, urls, page_id = compose_list(head, shown, rest, tail, [*general, *crzp_urls], mixed=False) return _with_token(_ok(question, engine, text, urls, base), wiki_context=wiki_context, question=question, entities=[entity(item, i) for i, item in enumerate(shown, 1)], page=page_id, crzp=part.get("crzp_token")) if wiki_ok and person and part["mode"] == "person": # A work the wiki answer already names is not repeated (its link stays in "Zdroje:"). from tools.answers.unified import catalog as wiki_catalog, _same_title titles = [w.get("title") for w in wiki_catalog()["works"].get(person, []) if w.get("title")] kept = [b for b in re.split(r"\n\s*\n", crzp_body) if not any(_same_title(t, (re.search(r"„([^“]+)“", b) or [None, ""])[1]) for t in titles)] crzp_body = "Ďalšie práce:\n\n" + "\n\n".join(kept) if kept else "" if not wiki_ok and (part["mode"] == "explicit" or wiki.get("status") == "error" or part.get("home_excluded")): text = crzp_body elif wiki_ok: text = wiki_body + ("\n\n" + crzp_body if crzp_body else "") else: text = "V ZP Wiki k tomu nie je záznam.\n\nV CRZP (Centrálny register záverečných prác):\n\n" + crzp_body urls = list(dict.fromkeys([*wiki_urls, *crzp_urls])) if urls: text += "\n\nZdroje: " + ", ".join(urls) result = _ok(question, engine, text, urls, base) return _with_token(result, wiki_context=wiki_context, entities=entities, question=question, crzp=part.get("crzp_token"), primary="crzp" if part["mode"] == "explicit" and not wiki_ok else "wiki") def _work_options(person: str, part: dict, low: str) -> list[dict]: """The person's works when the two sources hold works of different types and the question names none.""" if re.search(r"\b(?:bakalar\w*|diplom\w*|dizerta\w*|projekt\w*)\b", low): return [] wiki = wiki_works(person) wiki_types = {w["type"] for w in wiki.values()} thesis = {"diploma_thesis": "diplom", "bachelor_thesis": "bakalar"} crzp_only = [w for w in part.get("works", []) if w["type"] in thesis and w["type"] not in wiki_types] if not crzp_only: return [] wiki_options = [{"source": "wiki", "label": label + (f" „{w['topic']}“" if w.get("topic") else ""), "work": label} for label, w in wiki.items() if w["type"] in thesis][:2] return wiki_options + crzp_only if wiki_options else [] CHOICE_FILLERS = {"tak", "teda", "ta", "tu", "ten", "to", "tej", "chcem", "daj", "prosim", "cislo", "moznost", "a", "ano", "no", "hej", "mi", "tu", "pracu", "praca", "prace", "o", "tej", "tom", "volba", "ok", "jasne", "tú"} NUMBER_WORDS = {"jedna": 1, "jednotku": 1, "jednotka": 1, "prvu": 1, "prva": 1, "prvy": 1, "prve": 1, "prvej": 1, "dva": 2, "dve": 2, "dvojku": 2, "dvojka": 2, "druhu": 2, "druha": 2, "druhy": 2, "druhe": 2, "druhej": 2, "tri": 3, "trojku": 3, "trojka": 3, "tretiu": 3, "tretia": 3, "treti": 3, "tretej": 3} def _choice(low: str, options: list[dict]) -> dict | None: """"2", "tak 1", "tú prvú", "jednotku", "bakalársku", "z wiki" -> the chosen option.""" words = [w for w in re.findall(r"[a-z0-9]+", low) if w not in CHOICE_FILLERS] typed = [o for o in options if o.get("type")] if typed and len(words) <= 4: # options of different thesis types: "bakalársku", "tú diplomovú" from tools.answers.unified import asked_type kind = asked_type(low) same = [o for o in typed if o["type"] == kind] if kind and len(same) == 1: return same[0] if len(words) == 1: word = words[0] number = int(word) if word.isdigit() else NUMBER_WORDS.get(word) if number is not None: return options[number - 1] if 1 <= number <= len(options) else None if len(words) <= 4 and CHOICE_CRZP.search(low): return next((o for o in options if o["source"] == "crzp"), None) if len(words) <= 4 and CHOICE_WIKI.search(low): return next((o for o in options if o["source"] == "wiki"), None) return None def _answer_choice(question: str, chosen: dict, *, engine: str, limit: int, complete=None, deliver=None) -> dict: """The answer to the pending question about the chosen work (the caller binds it to the reply).""" from tools.answers.finalize import finalize deliver_answer = deliver if chosen.get("type"): # one of the person's theses: the same question about that thesis from tools.answers.unified import TYPES asked = question.rstrip(" ?.") + f" v {TYPES[chosen['type']][2]}?" result = _answer(asked, engine=engine, limit=limit, context_token=None, complete=complete, deliver=deliver) return {**result, "_asked_as": [asked, *result.pop("_asked_as", [])]} from tools.crzp.answering.wiki_bridge import wiki_person if chosen["source"] == "wiki": work = normalized(chosen["work"]) adjective = "diplomovej" if work.startswith("diplom") else "bakalárskej" if work.startswith("bakal") else "" asked = re.sub(r"\b(svojej\s+|jeho\s+|jej\s+)?(práci|práce|prácu)\b", lambda m: (m.group(1) or "") + adjective + " " + m.group(2), question, count=1) if adjective else question if asked == question: asked = question.rstrip(" ?.") + f" ({chosen['work']})?" result = deliver_answer(asked, engine=engine, limit=limit) if result.get("accepted_for_delivery") is True and isinstance(result.get("answer"), str): result = {**result, "answer": finalize(result["answer"], result.get("source_urls"))} return {**result, "_asked_as": [asked]} person = wiki_person(question) part = crzp_part(question, person, complete=complete) rows = [r for r in (part or {}).get("rows", []) if chosen.get("key") and _key_of(r) == chosen["key"]] if not part or not rows: return _ok(question, engine, "Vybranú prácu sa v CRZP nepodarilo znova načítať. Skús otázku zopakovať.", [], {}) from tools.crzp.answering import answer as crzp row = rows[0] detail = crzp.fetch.detail(row["sid"]) if row.get("sid") else None low = normalized(question) if METHODS.search(low): text = crzp_methods(person, row, detail) else: label, summary = crzp.summary(detail, complete) if detail else ("", "") text = f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{summary}" urls = [row["url"]] if row.get("url") else [] return _ok(question, engine, text + ("\n\nZdroj: " + urls[0] if urls else ""), urls, {}) def _key_of(row: dict) -> str: from tools.crzp.answering import answer as crzp return crzp.work_key(row)