diff --git a/app/routes.py b/app/routes.py index f9779ce..0a9ca3a 100644 --- a/app/routes.py +++ b/app/routes.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import logging +import os import re import sqlite3 from typing import ( @@ -171,7 +172,7 @@ def verified_answer(request: AnswerRequest) -> dict: if request.engine == "crzp": from tools.crzp.answering.answer import answer as crzp_answer result = crzp_answer(request.query, limit=request.limit, context_token=request.context_token) - else: + elif request.include_diagnostics: result = deliver_answer( request.query, engine=request.engine, @@ -179,6 +180,10 @@ def verified_answer(request: AnswerRequest) -> dict: include_diagnostics=request.include_diagnostics, **({'context_token':request.context_token} if request.context_token else {}), ) + else: # ZP Wiki and CRZP at the same time, one answer (tools/answers/combined.py) + from tools.answers.combined import answer as combined_answer + result = combined_answer(request.query, engine=request.engine, limit=request.limit, + context_token=request.context_token, deliver=deliver_answer) except Exception: logger.exception("Príprava overenej odpovede zlyhala.") return { diff --git a/conftest.py b/conftest.py index 29f34cd..fe4575d 100644 --- a/conftest.py +++ b/conftest.py @@ -11,6 +11,8 @@ os.environ.setdefault("RETRIEVAL_RERANKER", "off") # No background CRZP traffic from tests that start the API. os.environ.setdefault("CRZP_CHECK_INTERVAL_SECONDS", "0") os.environ.setdefault("CRZP_PREPARE", "0") +os.environ.setdefault("ANSWER_WITH_CRZP", "0") # tests answer from the wiki fixture only +os.environ.setdefault("CRZP_GRAPH_LINKS", "0") # Evaluation code lives outside Git (see local/OBSAH.md). Without it, tests that # import it at module level are skipped at collection instead of failing. diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 17285d4..14c6847 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -38,6 +38,9 @@ services: RETRIEVAL_LEMMATIZER: ${RETRIEVAL_LEMMATIZER:-stanza} RETRIEVAL_RERANKER: ${RETRIEVAL_RERANKER:-on} ANSWER_QUESTION_CATALOG: ${ANSWER_QUESTION_CATALOG:-1} + # Few glibc malloc arenas: many short worker threads (wiki + CRZP in parallel) otherwise keep + # a separate arena each and the process grows by ~100 MB per question on a 6 GB machine. + MALLOC_ARENA_MAX: "2" GIT_SSH_COMMAND: >- ssh -F /dev/null -i /run/secrets/zpwiki_ssh_key -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes diff --git a/packages/claude/client.py b/packages/claude/client.py index 211de1a..30078a7 100644 --- a/packages/claude/client.py +++ b/packages/claude/client.py @@ -16,7 +16,7 @@ import urllib.parse import urllib.request MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -ENGINES = ("rag", "graphrag", "crzp") +ENGINES = ("rag", "graphrag") # CRZP is part of both answers STATUSES = {"ok", "no_answer", "ambiguous", "error"} @@ -103,7 +103,7 @@ def error_result(reason): "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", - "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", + "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.", "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', @@ -217,14 +217,13 @@ def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=Non if ( engine not in ENGINES or mode not in {"answer", "retrieve"} - or engine == "crzp" and mode != "answer" or type(limit) is not int or not 1 <= limit <= 20 or not isinstance(query, str) ): raise ClientError("invalid_query") query = query.strip() - command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) + command = re.match(r"^/(rag|graphrag)(?=\s|$)", query, re.I) if command: if command[1].lower() != engine: raise ClientError("engine_mismatch") diff --git a/packages/claude/dist/crzp/manifest.json b/packages/claude/dist/crzp/manifest.json deleted file mode 100644 index aa7e029..0000000 --- a/packages/claude/dist/crzp/manifest.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "backend_required": true, - "default_mode": "verified_answer", - "engine": "crzp", - "entrypoint": "SKILL.md", - "files": { - "SKILL.md": "aa97aeecb3ac09e06a5bdb1e613d7ba67674b36349effc4220ffde98a506b77a", - "scripts/client.py": "13c7eb8ae62a2cec5b729518422eb4d102eb197da7cb4bc3aa521d1940ea6812", - "scripts/query.py": "642fd4646889caa839c2700883bbbe67f6d6edfcff7474780c8b29eeda200d19" - }, - "minimum_python": "3.10", - "name": "crzp", - "schema_version": 1, - "version": "1.13.0" -} diff --git a/packages/claude/dist/crzp/scripts/client.py b/packages/claude/dist/crzp/scripts/client.py deleted file mode 100644 index 211de1a..0000000 --- a/packages/claude/dist/crzp/scripts/client.py +++ /dev/null @@ -1,444 +0,0 @@ -"""Small, dependency-free client shared by Claude skills and the MCP bridge.""" - -from __future__ import annotations - -import argparse -import hashlib -from dataclasses import dataclass -import ipaddress -import json -import os -import re -from pathlib import Path -import stat -import urllib.error -import urllib.parse -import urllib.request - -MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -ENGINES = ("rag", "graphrag", "crzp") -STATUSES = {"ok", "no_answer", "ambiguous", "error"} - - -class ClientError(Exception): - """The message is a public reason code, never an exception from the transport.""" - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -@dataclass(frozen=True) -class Config: - url: str - key: str - timeout: float = 90 - allow_http: bool = False - - @classmethod - def load(cls): - values = {} - filename = os.getenv("ZP_AGENT_CONFIG", "").strip() - if filename: - try: - path = Path(filename).expanduser() - if os.name == "posix" and stat.S_IMODE(path.stat().st_mode) & 0o077: - raise ClientError("config_permissions_require_0600") - values = json.loads(path.read_text()) - if not isinstance(values, dict): - raise ValueError() - except ClientError: - raise - except (OSError, ValueError): - raise ClientError("invalid_configuration") from None - try: - config = cls( - os.getenv("ZP_AGENT_URL", values.get("backend_url", "")).strip().rstrip("/"), - os.getenv("ZP_AGENT_API_KEY", values.get("api_key", "")).strip(), - float(os.getenv("ZP_AGENT_TIMEOUT", values.get("timeout_seconds", 90))), - os.getenv("ZP_AGENT_ALLOW_HTTP", "").lower() in {"1", "true"}, - ) - config.validate() - return config - except (ValueError, TypeError, AttributeError): - raise ClientError("invalid_configuration") from None - - def validate(self): - try: - url = urllib.parse.urlsplit(self.url) - if ( - url.scheme not in {"http", "https"} - or not url.hostname - or url.username is not None - or url.password is not None - or url.query - or url.fragment - or any(c.isspace() for c in self.url) - ): - raise ValueError() - url.port - if url.scheme == "http": - local = url.hostname == "localhost" - try: - local = local or ipaddress.ip_address(url.hostname).is_loopback - except ValueError: - pass - if not local and not self.allow_http: - raise ValueError() - if not self.key or "\r" in self.key or "\n" in self.key or not 1 <= self.timeout <= 600: - raise ValueError() - except (ValueError, TypeError): - raise ClientError("invalid_configuration") from None - - -def error_result(reason): - messages = { - 'invalid_context':'Kontext predchádzajúcej otázky sa nepodarilo overiť. Zopakuj úplnú otázku s osobou, prácou a prípadným dátumom.', - 'stale_context':'Dáta alebo kontext konverzácie sa zmenili. Zopakuj úplnú otázku; starý výber stretnutia nepoužijem.', - 'expired_context':'Kontext predchádzajúcej otázky už vypršal. Zopakuj úplnú otázku.', - 'proposal_not_available':'Otázka žiada návrh možného prepojenia, nie iba existujúce fakty. Návrhový režim ešte nie je dostupný; výpis podkladov preto nevydávam za hotový návrh.', - "reference_unavailable": "Zdrojový repozitár sa nepodarilo verejne overiť. Odpoveď založenú iba na tomto odkaze neuvádzam.", - "unauthorized": "Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.", - "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", - "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", - "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", - "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", - "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", - "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", - 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', - "answer_rejected": "Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. " - "Neoverené údaje sa nezobrazia.", - "answer_unavailable": "Overenú odpoveď sa momentálne nepodarilo pripraviť. Skús to neskôr.", - "answer_capacity_exceeded": "Agent práve spracúva iné otázky. Skús to o chvíľu.", - "backend_busy": "Agent práve spracúva iné otázky. Skús to o chvíľu.", - "backend_timeout": "Vyhľadávanie neodpovedalo včas. Skús otázku zopakovať neskôr.", - "backend_unavailable": "Spojenie s backendom momentálne nie je dostupné. Skontroluj, či bežia služby a tunel.", - "retrieval_unavailable": "Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.", - "content_index_unavailable": "Obsahové vyhľadávanie momentálne nie je pripravené alebo sa jeho výstup " - "nepodarilo overiť. Treba overiť extrakciu pre aktuálnu verziu dát; " - "nejde o potvrdenie, že hľadané údaje neexistujú.", - } - return { - "status": "error", - "reason": reason, - "accepted_for_delivery": False, - "answer": messages.get(reason, "Overenú odpoveď sa momentálne nepodarilo získať."), - "sources": [], - "index_version": None, - "complete": False, - "truncated": False, - "truncation_reasons": [], - } - - -def request_json(config: Config, endpoint: str, body=None): - config.validate() - request = urllib.request.Request( - config.url + endpoint, - data=json.dumps(body, ensure_ascii=False).encode() if body is not None else None, - headers={ - "Authorization": "Bearer " + config.key, - "Accept": "application/json", - **({"Content-Type": "application/json"} if body is not None else {}), - }, - ) - try: - with urllib.request.build_opener(NoRedirect()).open( - request, timeout=config.timeout - ) as response: - raw = response.read(MAX_RESPONSE_BYTES + 1) - except urllib.error.HTTPError as exc: - code = exc.code - exc.close() - reason = { - 401: "unauthorized", - 403: "forbidden", - 404: "backend_endpoint_not_found", - 400: "invalid_query", - 422: "invalid_request", - 429: "backend_busy", - 503: "backend_unavailable", - }.get(code, "backend_http_error") - raise ClientError("redirect_not_allowed" if 300 <= code < 400 else reason) from None - except (urllib.error.URLError, TimeoutError, OSError) as exc: - timeout = isinstance(exc, TimeoutError) or isinstance( - getattr(exc, "reason", None), TimeoutError - ) - raise ClientError("backend_timeout" if timeout else "backend_unavailable") from None - if len(raw) > MAX_RESPONSE_BYTES: - raise ClientError("response_too_large") - try: - value = json.loads(raw) - if not isinstance(value, dict): - raise ValueError() - return value - except (ValueError, UnicodeError): - raise ClientError("invalid_backend_response") from None - - -def validate_metadata(value): - if ( - not {"complete", "truncated", "truncation_reasons", "index_version"} <= value.keys() - or value.get("complete") is not None - and type(value["complete"]) is not bool - or type(value.get("truncated")) is not bool - or value["truncated"] - and value.get("complete") is not False - or not isinstance(value.get("truncation_reasons"), list) - or not all(isinstance(x, str) for x in value["truncation_reasons"]) - or value["truncation_reasons"] - and not value["truncated"] - or value.get("index_version") is not None - and not isinstance(value["index_version"], dict) - ): - raise ClientError("invalid_backend_response") - - -def validated_delivery_check(value,query,engine): - check=value.get('delivery_check') - if check is None:return None - digest=hashlib.sha256(json.dumps([engine,query.strip()],ensure_ascii=False,separators=(',',':')).encode()).hexdigest() - if (not isinstance(check,dict) or set(check)!={'layer','code','request_fingerprint'} - or check.get('request_fingerprint')!=digest - or check.get('layer') not in {'delivery','scope','evidence','coverage','answer','verification','fulfilment','context','planning','retrieval','service','evidence_selection'} - or check.get('code') not in {'complete','partial','not_applicable','wrong_person','wrong_work_or_meeting', - 'wrong_period','mixed_snapshot','altered_evidence','missing_evidence','missing_person','invalid_citation','verification_unavailable', - 'request_not_fulfilled','invalid_review','undisclosed_gap','clarification_required','evidence_not_found','unavailable', - 'invalid_model_selection','model_unavailable'}): - raise ClientError('invalid_backend_response') - return check - - -def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=None, context_token=None): - """Return only approved answer fields, or explicitly marked retrieval context.""" - checked_delivery=None - try: - if ( - engine not in ENGINES - or mode not in {"answer", "retrieve"} - or engine == "crzp" and mode != "answer" - or type(limit) is not int - or not 1 <= limit <= 20 - or not isinstance(query, str) - ): - raise ClientError("invalid_query") - query = query.strip() - command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) - if command: - if command[1].lower() != engine: - raise ClientError("engine_mismatch") - query = query[command.end() :].strip() - if not 1 <= len(query) <= 500: - raise ClientError("invalid_query") - config = config or Config.load() - if context_token is not None and (mode!='answer' or not isinstance(context_token,str) or not 1<=len(context_token)<=16000): - raise ClientError('invalid_query') - endpoint = "/answer" if mode == "answer" else "/" + engine - body = {"query": query.strip(), "limit": limit} - if context_token is not None:body['context_token']=context_token - if mode == "answer" or engine == "rag": - body["include_diagnostics"] = False - if mode == "answer": - body["engine"] = engine - else: - body["published_only"] = True - value = request_json(config, endpoint, body) - if mode=='answer':checked_delivery=validated_delivery_check(value,query,engine) - if not isinstance(value.get("status"), str) or value["status"] not in STATUSES: - raise ClientError("invalid_backend_response") - if value["status"] == "error": - # Raw model output, diagnostics and rejected text must not escape. - reason = value.get("reason") - safe_reasons = { - 'verification_unavailable', - 'invalid_context','stale_context','expired_context','proposal_not_available', - "model_unavailable", - "answer_capacity_exceeded", - "answer_rejected", - "answer_unavailable", - "retrieval_unavailable", - "content_index_unavailable", - "reference_unavailable", - "crzp_unavailable", - } - raise ClientError( - reason - if isinstance(reason, str) and reason in safe_reasons - else "backend_answer_error" if mode == "answer" else "retrieval_unavailable" - ) - validate_metadata(value) - fields = { - "status", - "reason", - "index_version", - "complete", - "truncated", - "truncation_reasons", - } - if mode == "answer": - if not isinstance(value.get("query"), str): - raise ClientError("invalid_backend_response") - if value["query"] != body["query"] or value.get("engine") != engine: - raise ClientError("question_mismatch") - if value.get("accepted_for_delivery") is not True: - raise ClientError("answer_rejected") - if ( - not isinstance(value.get("answer"), str) - or not value["answer"].strip() - or not isinstance(value.get("source_urls"), list) - or not all(isinstance(x, str) for x in value["source_urls"]) - or not isinstance(value.get("claims"), list) - ): - raise ClientError("invalid_backend_response") - if "answer_coverage" in value: - coverage = value["answer_coverage"] - if ( - not isinstance(coverage, dict) - or coverage.get("scope") - not in ("fact", "overview", "assignment", "list", "count") - or coverage.get("status") - not in ("supported", "partial", "unknown", "not_found", "unavailable") - or any( - not isinstance(coverage.get(key), list) - or any(not isinstance(item, str) for item in coverage[key]) - for key in ["warnings", "limitations"] - ) - ): - raise ClientError("invalid_backend_response") - if "knowledge_version" in value and not isinstance(value["knowledge_version"], str): - raise ClientError("invalid_backend_response") - if 'conversation_context' in value: - context=value['conversation_context'] - if (not isinstance(context,dict) or not isinstance(context.get('token'),str) - or not 1<=len(context['token'])<=16000 - or any(not isinstance(context.get(k),list) for k in ['work_options','meeting_options'])): - raise ClientError('invalid_backend_response') - if 'resolved_query' in value and not isinstance(value['resolved_query'],str): - raise ClientError('invalid_backend_response') - if 'answer_kind' in value and value['answer_kind'] not in {'grounded_facts','recommendation'}: - raise ClientError('invalid_backend_response') - if 'recommendation_status' in value and value['recommendation_status'] not in {'accepted','insufficient_grounding','unavailable'}: - raise ClientError('invalid_backend_response') - fields |= { - "query", - "answer", - "source_urls", - "claims", - "count", - "accepted_for_delivery", - "answer_coverage", - } - fields.add("knowledge_version") - fields |= {'conversation_context','resolved_query','answer_kind','recommendation_status','delivery_check'} - else: - if not isinstance(value.get("sources"), list): - raise ClientError("invalid_backend_response") - if engine == "rag" and not isinstance(value.get("context"), str): - raise ClientError("invalid_backend_response") - if engine == "graphrag" and ( - not isinstance(value.get("facts"), list) or not isinstance(value.get("paths"), list) - ): - raise ClientError("invalid_backend_response") - fields |= { - "sources", - "context", - "facts", - "supporting_facts", - "paths", - "aggregation", - "constraints", - "answer_candidates", - "instructions", - } - result = {key: item for key, item in value.items() if key in fields} - return { - **result, - "engine": engine, - "output_kind": "verified_answer" if mode == "answer" else "retrieval_context", - **({"requires_final_answer_verification": True} if mode == "retrieve" else {}), - } - except ClientError as exc: - return {**error_result(str(exc)),**({'delivery_check':checked_delivery} if checked_delivery else {})} - - -def backend_health(config=None): - try: - value = request_json(config or Config.load(), "/health") - tools = value.get("tools") - if ( - type(value.get("ready")) is not bool - or not isinstance(tools, dict) - or any( - not isinstance(tools.get(engine), dict) - or type(tools[engine].get("ready")) is not bool - for engine in ("rag", "graphrag") - ) - ): - raise ClientError("invalid_backend_response") - return { - "status": "ok" if value["ready"] else "error", - "ready": value["ready"], - "index_version": value.get("index_version"), - **({"capabilities": value["capabilities"]} if isinstance(value.get("capabilities"), dict) else {}), - **({"answer_ready": value["answer_ready"]} if isinstance(value.get("answer_ready"), dict) else {}), - "tools": { - engine: { - k: v - for k, v in tools[engine].items() - if k in {"ready", "reason", "index_version"} - } - for engine in ENGINES - if isinstance(tools.get(engine), dict) and type(tools[engine].get("ready")) is bool - }, - } - except ClientError as exc: - return {**error_result(str(exc)), "ready": False} - - -def answer_for_presentation(result: dict) -> dict: - """Keep approved prose/caveats intact; retrieval telemetry is not answer prose. - - Called after query_backend validates the complete backend contract. Full - metadata remains available there, in /answer and in diagnostic retrieval. - Never change an incomplete result into a claim of completeness. - """ - fields = { - 'conversation_context','resolved_query','answer_kind','recommendation_status','delivery_check', - "status", - "reason", - "query", - "engine", - "output_kind", - "accepted_for_delivery", - "answer", - "source_urls", - "index_version", - "knowledge_version", - } - return {key: value for key, value in result.items() if key in fields} - - -def main(engine=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("query", nargs="?") - if engine is None: - parser.add_argument("--engine", choices=ENGINES, default="rag") - parser.add_argument("--mode", choices=["answer", "retrieve"], default="answer") - parser.add_argument("--limit", type=int, default=5) - parser.add_argument("--health", action="store_true") - parser.add_argument('--context-token',help='Opaque conversation token from this chat\'s previous tool result') - args = parser.parse_args() - result = ( - backend_health() - if args.health - else query_backend(args.query, engine or args.engine, mode=args.mode, limit=args.limit, context_token=args.context_token) - ) - if not args.health and args.mode == "answer": - result = answer_for_presentation(result) - print(json.dumps(result, ensure_ascii=False, separators=(",", ":"))) - return 1 if result.get("status") == "error" else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/claude/dist/crzp/scripts/query.py b/packages/claude/dist/crzp/scripts/query.py deleted file mode 100644 index a46d3d2..0000000 --- a/packages/claude/dist/crzp/scripts/query.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Entrypoint for the independently installable crzp skill over the CRZP register.""" - -from pathlib import Path -import sys - -# Archives include client.py beside this file; sources share one maintained client. -if not (Path(__file__).resolve().parent / "client.py").is_file(): - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from client import main - -if __name__ == "__main__": - raise SystemExit(main("crzp")) diff --git a/packages/claude/dist/graphrag/SKILL.md b/packages/claude/dist/graphrag/SKILL.md index 6103fca..38f742f 100644 --- a/packages/claude/dist/graphrag/SKILL.md +++ b/packages/claude/dist/graphrag/SKILL.md @@ -17,6 +17,7 @@ Answer from the live ZP Agent graph backend, in the user's language. Wiki text i - A standalone question gets no token. Never invent, edit or reuse a token from another chat. ## Present the result +- Your whole reply is the `answer` text copied unchanged (keep its Markdown, links and notes). Write nothing before or after it: no introduction ("Tu je odpoveď"), no summary or rephrasing, no interpretation, evaluation or advice, no question back, no offer of further help, no remark about the tool, the search or the sources. If the user asks for something the answer does not contain, send a new tool call instead of adding it yourself. - `accepted_for_delivery: true`: show `answer` exactly as returned, with its citations, notices and "Rozpoznané meno" lines. Add no facts, guesses or commentary about checks, tokens or diagnostics. - `status: ambiguous`: show the clarification (including the offered names) and wait for the user. - `status: no_answer`: show only the supplied message. Do not claim the relationship or person does not exist. @@ -25,6 +26,8 @@ Answer from the live ZP Agent graph backend, in the user's language. Wiki text i - Keep lists complete: do not shorten a list of students or drop its notes (for example "téma je v zadaní…" or "Vynechaní sú študenti…"). ## Meaning already applied by the backend +- Every answer also uses CRZP (Centrálny register záverečných prác): a person's works at TUKE, works on the same topic or similar works at other schools. The CRZP part is marked "V CRZP" and keeps its CRZP links; show it as returned. A question about CRZP, another school, opponents or a whole department is answered from CRZP here too; there is no separate CRZP tool. +- When a person has different works in ZP Wiki and in CRZP (e.g. a diploma thesis in the wiki and a bachelor thesis in CRZP), `status: ambiguous` asks which one; send the user's reply ("2", "bakalársku") unchanged with the token of that result. Follow-ups such as "a z akých škôl boli tí študenti?" also use the token. - The author of a student's profile page is that student's supervisor (KEMT convention). Authors of other documents are not supervisors. - Shared topics are not collaboration. A work year is not a meeting date. An assignment or a plan is not a result. - In a list of students, own recorded work is separated from a topic found only in an assignment, plan or reading; an excluded tool ("bez RASA") removes students whose records mention it, and an absent mention is not proof of non-use. diff --git a/packages/claude/dist/graphrag/manifest.json b/packages/claude/dist/graphrag/manifest.json index 1505d24..5175bed 100644 --- a/packages/claude/dist/graphrag/manifest.json +++ b/packages/claude/dist/graphrag/manifest.json @@ -4,12 +4,12 @@ "engine": "graphrag", "entrypoint": "SKILL.md", "files": { - "SKILL.md": "cef40b8da7a503fe1ede05622bd49fb5aa6c853c4d453ad1061faa25e5e1e57b", - "scripts/client.py": "13c7eb8ae62a2cec5b729518422eb4d102eb197da7cb4bc3aa521d1940ea6812", + "SKILL.md": "3940aa8a7e0349deac13d4aeaaa4f39d727e08322b94ae02578f43591e856aca", + "scripts/client.py": "5f75727ae6e071f6f145a3a61988ada0e716476e63d7f3c7b08786c210910f16", "scripts/query.py": "463460cd93cf2a584156df6aa571bcf98297e5a1a7258581d02b084b1e1a6180" }, "minimum_python": "3.10", "name": "graphrag", "schema_version": 1, - "version": "1.13.0" + "version": "2.0.0" } diff --git a/packages/claude/dist/graphrag/scripts/client.py b/packages/claude/dist/graphrag/scripts/client.py index 211de1a..30078a7 100644 --- a/packages/claude/dist/graphrag/scripts/client.py +++ b/packages/claude/dist/graphrag/scripts/client.py @@ -16,7 +16,7 @@ import urllib.parse import urllib.request MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -ENGINES = ("rag", "graphrag", "crzp") +ENGINES = ("rag", "graphrag") # CRZP is part of both answers STATUSES = {"ok", "no_answer", "ambiguous", "error"} @@ -103,7 +103,7 @@ def error_result(reason): "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", - "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", + "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.", "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', @@ -217,14 +217,13 @@ def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=Non if ( engine not in ENGINES or mode not in {"answer", "retrieve"} - or engine == "crzp" and mode != "answer" or type(limit) is not int or not 1 <= limit <= 20 or not isinstance(query, str) ): raise ClientError("invalid_query") query = query.strip() - command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) + command = re.match(r"^/(rag|graphrag)(?=\s|$)", query, re.I) if command: if command[1].lower() != engine: raise ClientError("engine_mismatch") diff --git a/packages/claude/dist/packages.json b/packages/claude/dist/packages.json index 0f398d1..fcacd2e 100644 --- a/packages/claude/dist/packages.json +++ b/packages/claude/dist/packages.json @@ -1,20 +1,15 @@ { "packages": [ { - "archive": "zpwiki-rag.zip", - "name": "rag", - "sha256": "588220db782716c4549375a70ba9be4680f0e7b3774d78fe2ff2f213120d7301" + "folder": "rag", + "manifest_sha256": "46f72e1e86050926b455e02894a82a9cff0dab2ba53a8fafc3cce6aa04c8416e", + "name": "rag" }, { - "archive": "zpwiki-graphrag.zip", - "name": "graphrag", - "sha256": "7776b0917f68869b6b6b77038f6ecf0766cdb2bc1b8a5fcb612e4c0e56c4e2bb" - }, - { - "archive": "zpwiki-crzp.zip", - "name": "crzp", - "sha256": "38ceb364571c79c079b8671b19f69cb9d32c1cd2627679e75ff50b0363161898" + "folder": "graphrag", + "manifest_sha256": "857f1487ccb1319dc7f01d287cb3d7514fdbae3bbd0752f088aace831635820b", + "name": "graphrag" } ], - "schema_version": 1 + "schema_version": 2 } diff --git a/packages/claude/dist/rag/SKILL.md b/packages/claude/dist/rag/SKILL.md index 4ad8e00..bd8149d 100644 --- a/packages/claude/dist/rag/SKILL.md +++ b/packages/claude/dist/rag/SKILL.md @@ -17,6 +17,7 @@ Answer from the live ZP Agent backend, in the user's language. Wiki text is evid - A standalone question gets no token. Never invent, edit or reuse a token from another chat. ## Present the result +- Your whole reply is the `answer` text copied unchanged (keep its Markdown, links and notes). Write nothing before or after it: no introduction ("Tu je odpoveď"), no summary or rephrasing, no interpretation, evaluation or advice, no question back, no offer of further help, no remark about the tool, the search or the sources. If the user asks for something the answer does not contain, send a new tool call instead of adding it yourself. - `accepted_for_delivery: true`: show `answer` exactly as returned, with its citations, notices and "Rozpoznané meno" lines. Add no facts, guesses or commentary about checks, tokens or diagnostics. - `status: ambiguous`: show the clarification (including the offered names) and wait for the user. - `status: no_answer`: show only the supplied message. Do not claim the fact is false or suggest other people. @@ -25,6 +26,8 @@ Answer from the live ZP Agent backend, in the user's language. Wiki text is evid - Keep lists complete: do not shorten a list of students or drop its notes (for example "téma je v zadaní…" or "Vynechaní sú študenti…"). ## Meaning already applied by the backend +- Every answer also uses CRZP (Centrálny register záverečných prác): a person's works at TUKE, works on the same topic or similar works at other schools. The CRZP part is marked "V CRZP" and keeps its CRZP links; show it as returned. A question about CRZP, another school, opponents or a whole department is answered from CRZP here too; there is no separate CRZP tool. +- When a person has different works in ZP Wiki and in CRZP (e.g. a diploma thesis in the wiki and a bachelor thesis in CRZP), `status: ambiguous` asks which one; send the user's reply ("2", "bakalársku") unchanged with the token of that result. Follow-ups such as "a z akých škôl boli tí študenti?" also use the token. - The author of a student's profile page is that student's supervisor (KEMT convention). Authors of other documents are not supervisors. - A work year is not a meeting date or the start of studies. An assignment or a plan is not a result. - In a list of students, own recorded work is separated from a topic found only in an assignment, plan or reading; an excluded tool ("bez RASA") removes students whose records mention it, and an absent mention is not proof of non-use. diff --git a/packages/claude/dist/rag/manifest.json b/packages/claude/dist/rag/manifest.json index 2c6a0ba..445cd06 100644 --- a/packages/claude/dist/rag/manifest.json +++ b/packages/claude/dist/rag/manifest.json @@ -4,12 +4,12 @@ "engine": "rag", "entrypoint": "SKILL.md", "files": { - "SKILL.md": "5f359a7cee5021778bc64e999cd2734427fca1674120cf87685bf1a80b376807", - "scripts/client.py": "13c7eb8ae62a2cec5b729518422eb4d102eb197da7cb4bc3aa521d1940ea6812", + "SKILL.md": "b64bd4b88857f0fc03bb1a3fbffc4cac8ef4ca6f00eacea4f9b4064ff3d55ee8", + "scripts/client.py": "5f75727ae6e071f6f145a3a61988ada0e716476e63d7f3c7b08786c210910f16", "scripts/query.py": "f482f2dc53430cbb383bce4fcc1cccaf580fc7b5a0c885a217fc40cf53713277" }, "minimum_python": "3.10", "name": "rag", "schema_version": 1, - "version": "1.13.0" + "version": "2.0.0" } diff --git a/packages/claude/dist/rag/scripts/client.py b/packages/claude/dist/rag/scripts/client.py index 211de1a..30078a7 100644 --- a/packages/claude/dist/rag/scripts/client.py +++ b/packages/claude/dist/rag/scripts/client.py @@ -16,7 +16,7 @@ import urllib.parse import urllib.request MAX_RESPONSE_BYTES = 2 * 1024 * 1024 -ENGINES = ("rag", "graphrag", "crzp") +ENGINES = ("rag", "graphrag") # CRZP is part of both answers STATUSES = {"ok", "no_answer", "ambiguous", "error"} @@ -103,7 +103,7 @@ def error_result(reason): "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", - "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", + "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.", "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', @@ -217,14 +217,13 @@ def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=Non if ( engine not in ENGINES or mode not in {"answer", "retrieve"} - or engine == "crzp" and mode != "answer" or type(limit) is not int or not 1 <= limit <= 20 or not isinstance(query, str) ): raise ClientError("invalid_query") query = query.strip() - command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) + command = re.match(r"^/(rag|graphrag)(?=\s|$)", query, re.I) if command: if command[1].lower() != engine: raise ClientError("engine_mismatch") diff --git a/packages/claude/dist/zpwiki-crzp.zip b/packages/claude/dist/zpwiki-crzp.zip deleted file mode 100644 index 80a1456..0000000 Binary files a/packages/claude/dist/zpwiki-crzp.zip and /dev/null differ diff --git a/packages/claude/dist/zpwiki-graphrag.zip b/packages/claude/dist/zpwiki-graphrag.zip deleted file mode 100644 index 9ef8db8..0000000 Binary files a/packages/claude/dist/zpwiki-graphrag.zip and /dev/null differ diff --git a/packages/claude/dist/zpwiki-rag.zip b/packages/claude/dist/zpwiki-rag.zip deleted file mode 100644 index 7a0db9b..0000000 Binary files a/packages/claude/dist/zpwiki-rag.zip and /dev/null differ diff --git a/packages/claude/mcp_server.py b/packages/claude/mcp_server.py index b9a349c..4f871a4 100644 --- a/packages/claude/mcp_server.py +++ b/packages/claude/mcp_server.py @@ -23,7 +23,7 @@ QUESTION_POLICY = ( "Load each tool definition with the host tool search when tools are deferred; use the exact loaded " "identifier and schema. Put the question into the query argument, never an argument named question. " "A tool-not-loaded or invalid-argument error is a host invocation error, not a DNS failure. " - "Send the user's latest question verbatim, removing only a leading /rag, /graphrag or /crzp command. Do not " + "Send the user's latest question verbatim, removing only a leading /rag or /graphrag command. Do not " "paraphrase, translate, correct names or add/drop conditions; the backend resolves misspelled and partial names. " "One call per question; several explicit questions in one message are separate calls in order. " "If the previous result contained conversation_context.token and the new message continues it (a name chosen " @@ -35,6 +35,9 @@ QUESTION_POLICY = ( ANSWER_PRESENTATION_POLICY = ( "When accepted_for_delivery is true, present answer exactly as returned, with its citations, notices and " "recognized-name lines; add no facts, guesses or commentary about checks, tokens or diagnostics. " + "The whole reply is that answer text unchanged: nothing before or after it (no introduction, summary, " + "rephrasing, evaluation, advice, question back or offer of further help); for anything the answer lacks, " + "call the tool again instead of adding it. " "For status ambiguous show the clarification and wait; for no_answer show only the supplied message and do not " "claim nonexistence; for error say a verified answer is unavailable and never write a replacement. After a " "timeout the user may repeat the same question; the backend then returns the finished answer. " @@ -81,20 +84,6 @@ async def zpwiki_graphrag(query: str, limit: int = 5, context_token: str | None ) -async def zpwiki_crzp(query: str, limit: int = 5, context_token: str | None = None) -> CallToolResult: - """Verified answer from CRZP (Central Register of Theses, all Slovak schools): works by title, student, - supervisor, opponent, school, year, type or topic; what a work is about, its keywords (tags), counts, - schools and related works. Metadata and abstracts only. - - Pass the full question in query, not question. Optional limit is an integer. - For a follow-up ("a v roku 2023?", "z akých škôl boli tí študenti?", "o čom je tá druhá?") copy - conversation_context.token from the last zpwiki_crzp result into context_token. - Present the accepted answer with its CRZP links; no_answer means no matching CRZP record was found. - """ - return tool_result(answer_for_presentation(await asyncio.to_thread(query_backend, query, "crzp", limit=limit, - **({'context_token':context_token} if context_token is not None else {})))) - - async def zpwiki_health() -> CallToolResult: """Read readiness and data versions of both tools without calling a language model.""" return tool_result(await asyncio.to_thread(backend_health)) @@ -104,15 +93,14 @@ def create_server(**kwargs): server = FastMCP( "ZP Wiki", instructions=( - "Use the tool selected by the user's /rag, /graphrag or /crzp command. Without a command use zpwiki_rag; " - "zpwiki_graphrag serves graph relationships, related topics and metadata counts; zpwiki_crzp searches " - "the national CRZP register of theses of all Slovak schools (use it for /crzp or when the user asks " - "about CRZP). " + "Use the tool selected by the user's /rag or /graphrag command. Without a command use zpwiki_rag; " + "zpwiki_graphrag serves graph relationships, related topics and metadata counts. Both also answer from " + "the national CRZP register of theses of all Slovak schools; CRZP parts are marked \"V CRZP\". " "Tools return backend-verified answers. " + QUESTION_POLICY + ANSWER_PRESENTATION_POLICY ), **kwargs, ) - for handler in (zpwiki_rag, zpwiki_graphrag, zpwiki_crzp, zpwiki_health): + for handler in (zpwiki_rag, zpwiki_graphrag, zpwiki_health): server.add_tool( handler, annotations=ToolAnnotations( diff --git a/test/test_academic.py b/test/test_academic.py index fd8fcb9..457590f 100644 --- a/test/test_academic.py +++ b/test/test_academic.py @@ -74,7 +74,7 @@ def test_latest_meeting_is_selected_by_date_even_when_profile_order_is_reversed( "Kedy mala Alena Nová posledné stretnutie a o čom bolo?", data, engine=engine ) assert r["status"] == "ok" and "2026-05-05" in r["answer"] - assert "Implementovaný" in r["answer"] and "Plán:" in r["answer"] + assert "Implementovaný" in r["answer"] and "Úloha zo stretnutia:" in r["answer"] assert "prvý návrh" not in r["answer"] assert all(e.get("date") == "2026-05-05" for e in r["evidence"]) diff --git a/test/test_answer_explanations.py b/test/test_answer_explanations.py index c199dca..270d9a9 100644 --- a/test/test_answer_explanations.py +++ b/test/test_answer_explanations.py @@ -60,7 +60,7 @@ def test_meeting_prose_retains_planned_vs_executed_without_copying_imperatives() 'text':'Má vyhodnotiť systém.' if r['phase']=='planned' else 'Funguje agent RAG.'} for r in payload['evidence']]} return {'choices':[{'message':{'content':json.dumps(value)}}]} out=s.synthesize(result,plan,complete=model,diagnostics=True) - assert out['synthesis']['status']=='accepted' and 'Ďalší postup: Má vyhodnotiť systém.' in out['answer'] + assert out['synthesis']['status']=='accepted' and 'Úlohy: Má vyhodnotiť systém.' in out['answer'] assert 'Vyhodnoťte' not in out['answer'] and 'Vyhodnotil' not in out['answer'] diff --git a/test/test_claude_packages.py b/test/test_claude_packages.py index d0a3d81..5d38f0a 100644 --- a/test/test_claude_packages.py +++ b/test/test_claude_packages.py @@ -410,8 +410,8 @@ def test_config_file_permissions_and_environment_override(tmp_path, monkeypatch) @pytest.mark.parametrize("engine", ["rag", "graphrag"]) def test_zip_is_reproducible_and_runs_without_repository(tmp_path, backend, engine): - one = build_package(engine, tmp_path / "one") - two = build_package(engine, tmp_path / "two") + one = build_package(engine, tmp_path / "one", tmp_path / "one") + two = build_package(engine, tmp_path / "two", tmp_path / "two") assert one["sha256"] == two["sha256"] archive_path = tmp_path / "one" / one["archive"] folder = tmp_path / "installed" @@ -456,18 +456,20 @@ def test_zip_is_reproducible_and_runs_without_repository(tmp_path, backend, engi assert raw["complete"] is False and raw["truncated"] is True -def test_committed_archives_match_current_sources(tmp_path): - manifest = json.loads((ROOT / "packages/claude/dist/packages.json").read_text()) +def test_committed_skill_folders_match_current_sources(tmp_path): + dist = ROOT / "packages/claude/dist" + manifest = json.loads((dist / "packages.json").read_text()) + assert not list(dist.glob("*.zip")) # archives are made on request only (--zip) for row in manifest["packages"]: engine = row["name"] rebuilt = build_package(engine, tmp_path) - actual = ROOT / "packages/claude/dist" / row["archive"] - assert hashlib.sha256(actual.read_bytes()).hexdigest() == rebuilt["sha256"] == row["sha256"] - # The unpacked folder in dist/ holds exactly the archive's files. - import zipfile - with zipfile.ZipFile(actual) as archive: - for member in archive.namelist(): - assert (ROOT / "packages/claude/dist" / member).read_bytes() == archive.read(member), member + assert rebuilt["manifest_sha256"] == row["manifest_sha256"] + built = sorted(p.relative_to(tmp_path / engine) for p in (tmp_path / engine).rglob("*") if p.is_file()) + kept = sorted(p.relative_to(dist / engine) for p in (dist / engine).rglob("*") + if p.is_file() and "__pycache__" not in p.parts) + assert built == kept + for path in built: + assert (dist / engine / path).read_bytes() == (tmp_path / engine / path).read_bytes(), path def test_mcp_stdio_initialization_discovery_call_and_rejection(backend): @@ -497,7 +499,6 @@ def test_mcp_stdio_initialization_discovery_call_and_rejection(backend): assert {row.name for row in listed.tools} == { "zpwiki_rag", "zpwiki_graphrag", - "zpwiki_crzp", "zpwiki_health", } for tool in listed.tools: @@ -634,7 +635,7 @@ def test_remote_mcp_authentication_and_full_protocol(backend, monkeypatch): ) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() - assert len((await session.list_tools()).tools) == 4 + assert len((await session.list_tools()).tools) == 3 for engine in ["rag", "graphrag"]: result = await session.call_tool( "zpwiki_" + engine, {"query": "Práca Evy Novej?"} @@ -817,7 +818,7 @@ def test_oauth_discovery_and_full_flow_with_mcp(oauth_app): ) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() - assert len((await session.list_tools()).tools) == 4 + assert len((await session.list_tools()).tools) == 3 health = await session.call_tool("zpwiki_health") assert not health.isError and health.structuredContent["ready"] result = await session.call_tool( diff --git a/test/test_combined_answers.py b/test/test_combined_answers.py new file mode 100644 index 0000000..4734116 --- /dev/null +++ b/test/test_combined_answers.py @@ -0,0 +1,115 @@ +"""ZP Wiki + CRZP in one answer: order, labels, clarification, follow-ups, CRZP timeout.""" +import time + +import pytest + +from tools.answers import combined as C + +WIKI = {"status": "ok", "reason": None, "accepted_for_delivery": True, "claims": [], "index_version": {"version": "v"}, + "complete": True, "truncated": False, "truncation_reasons": [], + "answer": "**Ján Pták — Diplomová práca 2027**\n\nVedúci práce: Daniel Hladek.\n\nZdroje: https://zp.test/jan", + "source_urls": ["https://zp.test/jan"], + "delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": "x"}} + + +@pytest.fixture +def combo(monkeypatch): + monkeypatch.setenv("SEARCH_API_KEY", "combined-test-key") + monkeypatch.setenv("ANSWER_WITH_CRZP", "1") + state = {"part": None, "match": {"name": "Ján Pták", "student": True}} + monkeypatch.setattr("tools.crzp.answering.wiki_bridge.wiki_match", lambda q, db_file=None: state["match"]) + monkeypatch.setattr(C, "crzp_part", lambda q, person, complete=None, student=True: state["part"]) + state["deliver"] = lambda q, **kw: dict(WIKI) + return state + + +def ask(combo, question, token=None): + return C.answer(question, engine="rag", context_token=token, deliver=lambda q, **kw: combo["deliver"](q, **kw)) + + +def test_wiki_first_then_the_marked_crzp_part(combo): + combo["part"] = {"mode": "person", "text": "- bakalárska práca 2025 „IoT“ — vedúci Pavol Galajda; oponent Daniel Hládek", + "rows": [], "urls": ["https://opac.test/1"], "people": [], "works": []} + result = ask(combo, "Kto vedie Jána Ptáka?") + text = result["answer"] + assert text.index("Vedúci práce: Daniel Hladek") < text.index("**V CRZP**") < text.index("Pavol Galajda") + assert result["source_urls"] == ["https://zp.test/jan", "https://opac.test/1"] and result["status"] == "ok" + import packages.claude.client as client + client.validate_metadata(result) + assert client.validated_delivery_check(result, "Kto vedie Jána Ptáka?", "rag") + + +def test_without_crzp_the_wiki_answer_is_unchanged_but_for_wording(combo): + result = ask(combo, "Kedy mal Ján Pták posledné stretnutie?") + assert "V CRZP" not in result["answer"] and result["answer"].startswith("**Ján Pták") + + +def test_a_work_only_crzp_has_is_asked_for_and_the_choice_continues(combo, monkeypatch): + combo["part"] = {"mode": "person", "text": "x", "rows": [], "urls": [], "people": [], + "works": [{"source": "crzp", "label": "bakalárska práca 2025 „IoT“", "type": "bachelor_thesis", "key": "k"}]} + monkeypatch.setattr(C, "wiki_works", lambda person: {"Diplomová práca 2027": {"type": "diploma_thesis", "topic": "Agent"}}) + asked = ask(combo, "Aké technológie využíva Ján Pták vo svojej práci?") + assert asked["status"] == "ambiguous" and "1. Diplomová práca 2027 „Agent“ (ZP Wiki)" in asked["answer"] + assert "2. bakalárska práca 2025 „IoT“ (CRZP)" in asked["answer"] + seen = [] + combo["deliver"] = lambda q, **kw: (seen.append(q), dict(WIKI))[1] + ask(combo, "1", asked["conversation_context"]["token"]) + assert seen == ["Aké technológie využíva Ján Pták vo svojej diplomovej práci?"] + chosen = [] + monkeypatch.setattr(C, "_answer_choice", lambda q, c, **kw: chosen.append(c) or dict(WIKI)) + ask(combo, "tú bakalársku", asked["conversation_context"]["token"]) + assert chosen[0]["source"] == "crzp" + + +def test_schools_of_the_people_of_the_previous_answer(combo): + combo["part"] = {"mode": "similar", "text": "- Maksym Zatirka — Technická univerzita v Košiciach: „X“", "rows": [], + "urls": [], "people": [("Eva Malá", "Slovenská technická univerzita v Bratislave")]} + first = ask(combo, "Kto pracoval na podobnej téme ako Ján Pták?") + combo["match"] = None + schools = ask(combo, "a z akých škôl boli tí študenti?", first["conversation_context"]["token"]) + assert "- Ján Pták — Technická univerzita v Košiciach (KEMT FEI)" in schools["answer"] + assert "- Eva Malá — Slovenská technická univerzita v Bratislave" in schools["answer"] + + +def test_a_slow_crzp_does_not_hold_the_wiki_answer(combo, monkeypatch): + monkeypatch.setattr(C, "CRZP_SECONDS", 0.5) + monkeypatch.setattr(C, "crzp_part", lambda *a, **k: (time.sleep(4), None)[1]) + started = time.monotonic() + result = ask(combo, "Kto vedie Jána Ptáka?") + assert result["status"] == "ok" and time.monotonic() - started < 2.0 + + +def test_a_forged_combined_token_is_refused(combo): + first = ask(combo, "Kto vedie Jána Ptáka?") + token = first["conversation_context"]["token"] + assert ask(combo, "a z akých škôl?", token[:-3] + "abc")["reason"] == "invalid_context" + + +def test_crzp_people_and_keywords_link_to_the_wiki_graph_by_code(): + from tools.crzp.storage.graph import link_pairs + rows = [{"school": "Technická univerzita v Košiciach", "authors": ["Pták, Ján"], "supervisors": ["Galajda, Pavol"], + "opponents": [], "keywords": ["Speech Recognition", "iot"]}, + {"school": "Iná univerzita", "authors": ["Pták, Jan"], "supervisors": [], "opponents": [], "keywords": []}] + people, keywords = link_pairs(rows, ["Ján Pták", "Daniel Hladek"], ["rozpoznávanie reči", "Dialógový systém"]) + assert people == [{"crzp": "Pták, Ján", "wiki": "Ján Pták"}] # only at TUKE, the namesake elsewhere is not linked + assert keywords == [{"key": "speech recognition", "wiki": "rozpoznávanie reči"}] # through the reviewed synonyms + + +def test_the_wiki_question_which_person_wins_over_crzp(combo): + combo["match"] = None + combo["part"] = {"mode": "explicit", "status": "ambiguous", "text": "Menu „Ján“ zodpovedá v CRZP…", "rows": [], + "urls": [], "people": []} + combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "ambiguous_person", + "answer": "Ktorého Jána myslíš? Ján Holp, Ján Pták."} + result = ask(combo, "Na čom pracoval Ján?") + assert result["answer"] == "Ktorého Jána myslíš? Ján Holp, Ján Pták." + combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "missing_subject", + "answer": "Uveď meno študenta."} + combo["part"] = {**combo["part"], "status": "ok", "text": "Školitelia prác …"} + assert "Školitelia prác" in ask(combo, "Kto viedol najviac prác o strojovom učení?")["answer"] + + +def test_an_unknown_person_goes_to_crzp_only_for_a_thesis_question(): + assert C.THESIS.search(C.normalized("O čom je bakalárska práca Jána Nováka?")) + assert not C.THESIS.search(C.normalized("Na čom pracuje Peter Novák?")) + assert not C.ABOUT.search(C.normalized("Čomu sa venuje Jakub Schwarc?")) diff --git a/test/test_content_selection_ids.py b/test/test_content_selection_ids.py index adb649c..7e89c1f 100644 --- a/test/test_content_selection_ids.py +++ b/test/test_content_selection_ids.py @@ -280,3 +280,76 @@ def test_graph_narrative_request_uses_shared_plan_and_content_evidence(monkeypat assert result['status']=='ok' and result['accepted_for_delivery'] assert all(name in result['answer'] for name in names) assert 'problém s modelom' in result['answer'] and 'experiment' in result['answer'] + + +def test_a_status_record_naming_tools_answers_which_technologies_were_used(): + from tools.answers.content_selection import facets_for, named_tools + q = "Aké technológie použil Ján Pták vo svojej diplomovej práci?" + assert "methods" in facets_for("- Sú Skills aj MCP server. RAG vie v ZPwiki vyhľadávať.", "progress", q) + assert "methods" in facets_for("- začíname pracovať na GraphRAG a Neo4J", "progress", q) + assert "methods" in facets_for("- Funguje vektorové a textové vyhľadávanie.", "result", q) + assert "methods" not in facets_for("- Vytvorte nasadenie agenta pomocou DockerCompose.", "plan", q) + assert {"MCP", "RAG", "GraphRAG", "Neo4J", "Fast120", "me5mini"} <= set( + named_tools("MCP, RAG, GraphRAG, Neo4J, Fast120 a me5mini")) and not named_tools("Stretnutie 5.11.2026") + + +def test_a_meeting_question_is_not_narrowed_to_a_guessed_aspect(): + from tools.answers.question_catalog import Route + from tools.answers.question_plan import direct_spec + from tools.retrieval.person_names import resolve_person_names + catalog = [{"name": "Jakub Schwarc", "kind": "Person", "source_document": "pages/students/2022/jakub_schwarc/README.md"}] + for question, facets in [("Kedy mal Jakub Schwarc posledné stretnutie a čo na ňom riešil?", []), + ("Aké výsledky mal Jakub Schwarc na poslednom stretnutí?", ["results"])]: + route = Route(type="meeting", mode="meeting", facets=["results"], meeting="latest") + spec = direct_spec(question, resolve_person_names(question, catalog), route=route) + assert spec.facets == facets, question + + +def test_an_aspect_named_in_the_question_wins_over_the_classifier(): + from tools.answers.question_catalog import Route + from tools.answers.question_plan import direct_spec + from tools.retrieval.person_names import resolve_person_names + catalog = [{"name": "Jakub Schwarc", "kind": "Person", "source_document": "pages/students/2022/jakub_schwarc/README.md"}] + question = "Aké úlohy dostal Jakub Schwarc?" + route = Route(type="work_details", mode="facets", facets=["assignment"]) + assert direct_spec(question, resolve_person_names(question, catalog), route=route).facets == ["plans"] + + +def test_final_pass_writes_full_sources_and_drops_system_notices(): + from tools.answers.finalize import finalize + text = ("**Ján Pták — Diplomová práca 2027**\n\nNázov / téma zo zdroja: Agent\n\nPoužíva RAG. [1]\n\n" + "Vedúci je určený podľa autora profilovej stránky študenta v ZP Wiki.\n\n" + "Profil obsahuje aj záznamy bez úplného dátumu; nemožno spoľahlivo určiť, či sú novšie. Rok nedopĺňam z roku práce.\n\n" + "Zdroje: [1](https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak") + out = finalize(text) + assert "[1]" not in out and out.endswith("Zdroje: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak") + assert "Téma: Agent" in out and "Vedúci je určený" not in out and "Profil obsahuje" not in out + only_missing = "Zadanie a ciele:\n\nJakub Schwarc — Diplomová práca 2027: zadanie a ciele: v podkladoch chýbajú doložené údaje." + assert finalize(only_missing) == "ZP Wiki k tomu nemá záznam." + + +def test_final_pass_simplifies_technical_bullets_and_collapses_assignment_only_students(): + from tools.answers.finalize import finalize + text = ("**Samuel Džurina**\n\n- Bakalárska práca 2026; priebežný stav; záznam 2025-05-13: Chatbot funguje.\n\n" + "**Martin Jancura**\n\nMartin Jancura — téma je v zadaní, téme alebo pláne práce; zaznamenaný výsledok k nej sa nenašiel.\n\n" + "- Diplomový projekt; podklad: ## Diplomový projekt 2\n\n" + "**Dominik Nagy**\n\n- Diplomová práca 2022; podklad: *Meno vedúceho*: Ing. Daniel Hládek, PhD.\n\n" + "- Diplomová práca 2022; téma alebo názov: *Názov diplomovej práce*: Neurónový strojový preklad\n\n" + "Zdroje: https://a.test/x, https://a.test/y") + out = finalize(text) + assert "- Bakalárska práca 2026 (2025-05-13): Chatbot funguje." in out + assert "- Diplomová práca 2022: Neurónový strojový preklad" in out and "Hládek" not in out and "##" not in out + assert out.count("Martin Jancura") == 1 and "Téma je len v zadaní alebo pláne, bez zaznamenaného výsledku: Martin Jancura." in out + + +def test_related_topics_find_inflected_phrases_and_synonyms(): + from tools.retrieval.synonyms import base_name, variants_pattern + from tools.answers.context_policy import normalized + assert base_name("rozpoznávaním reči") == base_name("speech recognition") == "rozpoznávanie reči" + assert variants_pattern("speech recognition").search(normalized("Funguje automatické rozpoznávanie reči.")) + assert base_name("Python") == "Python" + + +def test_vyuziva_asks_for_methods_like_pouziva(): + from tools.answers.question_intent import requested_facets + assert requested_facets("Aké technológie využíva Ján Pták vo svojej diplomovej práci?") == ["methods"] diff --git a/test/test_conversation_context.py b/test/test_conversation_context.py index 4d5de90..620cafe 100644 --- a/test/test_conversation_context.py +++ b/test/test_conversation_context.py @@ -244,3 +244,16 @@ def test_client_transports_context_without_rewriting_query(monkeypatch): result=client.query_backend('Daj obe.','rag',config=client.Config('http://localhost:8000','test'),context_token='opaque') assert captured['query']=='Daj obe.' and captured['context_token']=='opaque' assert client.answer_for_presentation(result)['conversation_context']['token']=='opaque' + + +@pytest.mark.parametrize('q',['Kto bol jeho vedúci?','Aké technológie použil?','Kedy mal posledné stretnutie?', + 'Aké výsledky dosiahol tento študent?']) +def test_pronoun_or_aspect_only_question_continues_with_the_previous_person(data,q): + r=resolve_question(q,token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION) + assert 'Ján Holp' in r.query and r.clarification is None + + +@pytest.mark.parametrize('q',['Kto robil chatbot?','Kto pracoval s Cassandrou?']) +def test_a_new_topic_is_not_given_the_previous_person(data,q): + r=resolve_question(q,token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION) + assert 'Ján Holp' not in r.query diff --git a/tools/answers/combined.py b/tools/answers/combined.py new file mode 100644 index 0000000..0047362 --- /dev/null +++ b/tools/answers/combined.py @@ -0,0 +1,600 @@ +"""ZP Wiki and CRZP in one answer for /rag and /graphrag. + +Every question is answered from ZP Wiki (the normal RAG / GraphRAG path) and, +at the same time, from CRZP when CRZP can add something (see crzp_plan): +- a person ZP Wiki knows: that person's works in CRZP at Technická univerzita + v Košiciach (supervisor, opponent, abstract, keywords the wiki may lack); +- a topic or "similar works": CRZP works on the same topic, with their schools; +- a question naming CRZP, another school or opponents: CRZP is the main source. +Meetings, tasks, progress and results stay ZP Wiki only. The wiki part comes +first, the CRZP part is marked with its source; nothing is mixed into one fact. + +When a person has different works in the two sources (a diploma thesis in the +wiki, a bachelor thesis in CRZP) and the question does not say which one, the +answer asks; the reply ("2", "bakalársku") continues through a signed token +that also carries the wiki's own token and the people of the last answer +(for "a z akých škôl boli tí študenti?"). +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import os +import re +import threading +import time +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout +from functools import lru_cache + +from tools.answers.context_policy import normalized +from tools.answers.delivery_contract import fingerprint + +CRZP_SECONDS = float(os.getenv("CRZP_COMBINED_SECONDS", "45")) +LIFETIME = 24 * 3600 +PREFIX = "zc1." +TUKE = "Technická univerzita v Košiciach" +WIKI_SCHOOL = "Technická univerzita v Košiciach (KEMT FEI)" +# What only the wiki records: CRZP is not asked. +WIKI_ONLY = re.compile(r"\b(?:stretnut\w*|pokrok\w*|progres\w*|uloh\w*|zadan\w*|plan\w*|termin\w*|tyzd\w*|" + r"repozitar\w*|git\w*|stav\w*|poznam\w*|konzult\w*|zaciat\w*|studi\w*|obhaj\w*|ukonc\w*|" + r"dokonc\w*|uspes\w*|statnic\w*|hodnot\w*|znamk\w*|problem\w*|vtedy|posledn\w*)\b") +EXPLICIT = re.compile(r"\b(?:crzp|registr\w*|register|celoslovensk\w*|na slovensku|inych skol|inej skole|" + r"vsetkych skol|ostatnych skol|oponent\w*|oponoval\w*)\b") +METHODS = re.compile(r"\b(?:technolog\w*|nastroj\w*|metod\w*|pouziv\w*|pouzil\w*|vyuziv\w*|vyuzil\w*|framework\w*)\b") +ABOUT = re.compile(r"\b(?:o com|zhrn\w*|abstrakt\w*|ciel\w*)\b") # not "čomu sa venuje": the current work +# A question about a thesis (not "na čom pracuje", which CRZP's submitted works cannot answer). +THESIS = re.compile(r"\b(?:prac(?:a|e|u|i|ou|ach|ami)?|bakalar\w*|diplom\w*|dizerta\w*|habilita\w*|tem(?:a|u|y|ou)|" + r"oponent\w*|vedu\w*|skolitel\w*|abstrakt\w*|klucov\w*|tagy?)\b") +WIKI_CLARIFICATION_YIELDS = {"missing_subject", "unsupported_content_question"} +REFUSED = {"request_out_of_scope", "irrelevant", "unsupported_request", "out_of_scope"} +RELATED = re.compile(r"\b(?:suvis\w*|pribuzn\w*)\b") +SIMILAR = re.compile(r"\b(?:podobn\w*|suvisiac\w*|pribuzn\w*|spolupra\w*|rovnak\w* tem\w*)\b") +TOPIC_LIST = re.compile(r"^\W*(?:kto|ktor\w*|ak\w*)\b") +SCHOOLS = re.compile(r"\b(?:z|zo|na) (?:akych|akej|ktorych|ktorej) (?:skol|univerzit|fakult)\w*|\bkde (?:studoval\w*|studuj\w*)" + r"|\b(?:ktor|ak)\w* (?:skol|univerzit)\w*") +CHOICE_WIKI = re.compile(r"\b(?:wiki|diplom\w*|dp)\b") +CHOICE_CRZP = re.compile(r"\b(?:crzp|bakalar\w*|bp)\b") + + +_shared = {"pool": None} +_shared_lock = threading.Lock() + + +def _pool() -> ThreadPoolExecutor: + """One pool for all questions (no new threads per request).""" + with _shared_lock: + if _shared["pool"] is None: + _shared["pool"] = ThreadPoolExecutor(max_workers=int(os.getenv("ANSWER_COMBINED_WORKERS", "6")), + thread_name_prefix="answer") + return _shared["pool"] + + +def enabled() -> bool: + return os.getenv("ANSWER_WITH_CRZP", "1") != "0" + + +# ------------------------------------------------------------------ token + +def _key() -> bytes | None: + secret = os.getenv("SEARCH_API_KEY", "") + return hashlib.sha256(("zp-combined-v1\0" + secret).encode()).digest() if secret else None + + +def _b64(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + +def seal(payload: dict) -> str | None: + key = _key() + if key is None: + return None + body = _b64(json.dumps({**payload, "expires": int(time.time()) + LIFETIME}, ensure_ascii=False, + separators=(",", ":")).encode()) + token = PREFIX + body + "." + _b64(hmac.digest(key, body.encode(), "sha256")) + return token if len(token) <= 16000 else None + + +def unseal(token: str | None) -> dict | None: + """The combined context, or None for a token that is not ours (a plain wiki token).""" + if not isinstance(token, str) or not token.startswith(PREFIX) or _key() is None: + return None + try: + body, signature = token[len(PREFIX):].split(".") + expected = hmac.digest(_key(), body.encode(), "sha256") + if not hmac.compare_digest(base64.b64decode(signature + "=" * (-len(signature) % 4), altchars=b"-_"), expected): + raise ValueError + value = json.loads(base64.b64decode(body + "=" * (-len(body) % 4), altchars=b"-_")) + if value.get("expires", 0) < time.time(): + return {"error": "expired_context"} + return value + except (ValueError, TypeError, UnicodeError): + return {"error": "invalid_context"} + + +# ------------------------------------------------------------------ wiki facts + +_index_cache: dict = {} +_index_lock = threading.Lock() + + +def _index(): + """The published knowledge index, reloaded only when its version changes.""" + from tools.knowledge.store import load_index + from tools.rag.common import DB_FILE + from tools.sync.index_state import read_version + version = read_version(DB_FILE.parent) + key = (version or {}).get("version") + with _index_lock: + if _index_cache.get("key") != key: + _index_cache.update(key=key, index=load_index(DB_FILE.parent, version)) + return _index_cache["index"] + + +def wiki_units(person: str) -> list[dict]: + try: + return [u for u in (_index() or {}).get("units", []) if u.get("person") == person] + except Exception: + return [] + + +def wiki_works(person: str) -> dict[str, dict]: + """work label -> {"type", "topic"} of the person's ZP Wiki works.""" + works = {} + for unit in wiki_units(person): + label = unit.get("work_label") + if not label: + continue + work = works.setdefault(label, {"type": unit.get("work_type"), "topic": None}) + if unit.get("kind") == "topic" and not work["topic"] and not unit["text"].strip().endswith(":"): + work["topic"] = re.sub(r"^\s*(?:Téma|Názov)\s*:\s*", "", unit["text"]).strip() + return works + + +FRAMEWORKS = {"python", "fastapi", "flask", "django", "streamlit", "react", "docker", "kubernetes", "git", "github", + "gitlab", "pandas", "numpy", "jupyter", "colab", "linux", "javascript", "typescript", "java", "html", "css", + "llm", "ai", "nlp", "veľký jazykový model", "umelá inteligencia", "zpwiki", "openwebui", "skills", "skill"} + + +def wiki_terms(person: str) -> list[str]: + """The person's most specific topics and tools (reviewed topic groups first, frameworks never): + the words to look for in CRZP to find similar works.""" + from tools.answers.content_selection import named_tools + from tools.retrieval.synonyms import base_name, group_of + counts: dict[str, float] = {} + for unit in wiki_units(person): + if unit.get("phase") not in {"result", "progress", "assignment"} and unit.get("kind") != "topic": + continue + for concept in unit.get("concepts") or []: + if not isinstance(concept, str): + continue + if len(group_of(concept)) > 1: + name, weight = base_name(concept), 2.0 + elif named_tools(concept) and len(concept.split()) <= 2 and not re.search(r"[./]", concept): + name, weight = concept, 1.0 + else: + continue + if normalized(name) not in FRAMEWORKS: + counts[name] = counts.get(name, 0) + weight + return sorted(counts, key=lambda n: (-counts[n], n))[:4] + + +def wiki_profile(person: str) -> str: + """Topic titles and terms of the person's works: the text similar CRZP works are ranked against.""" + works = wiki_works(person) + return ". ".join([*(w["topic"] for w in works.values() if w.get("topic")), ", ".join(wiki_terms(person))]) + + +# ------------------------------------------------------------------ CRZP part + +def crzp_plan(question: str, person: str | None, complete=None, student: bool = True): + """(mode, CrzpQuery) or None when CRZP adds nothing to this question.""" + from tools.crzp.answering.understand import Person, understand + low = normalized(question) + explicit = bool(EXPLICIT.search(low)) + if WIKI_ONLY.search(low) and not explicit: + return None + query = understand(question, complete=complete) + if query.intent == "refused": + return None + other_school = query.school not in (None, "tuke") or explicit + if person and not other_school: + if SIMILAR.search(low) and student: + return "similar", query + role = query.people[0].role if query.people else "any" + if student: + role = "author" if role == "author" else "any" # "Kto vedie Jána Ptáka?": his own works + elif role not in ("supervisor", "opponent"): + role = "supervisor" # a supervisor's works are those he or she led + query.people = [Person(name=person, role=role, typed=person)] + query.school, query.unit, query.from_model, query.title = "tuke", None, True, None + query.topics, query.alternatives = [], [] + return "person", query + if query.people and not person and query.search_text() and THESIS.search(low): + return "explicit", query # a person ZP Wiki does not know: CRZP knows the theses of all schools + if query.topics and not query.people and RELATED.search(low): + query.intent = "keywords" # "Ktoré témy súvisia s X?": the tags of CRZP works about X + return "explicit", query + if query.topics and not query.people and query.intent in ("relation", "count", "keywords", "schools"): + return "explicit", query # "Kto viedol najviac prác o X?", "Koľko prác o X…": computed over CRZP + if (other_school or ((query.school or query.unit) and not query.people)) and query.search_text(): + return "explicit", query # another school, CRZP itself, opponents, or a school/department as a whole + if query.topics and not query.people and (TOPIC_LIST.search(low) or SIMILAR.search(low)): + return "topic", query + return None + + +def clean_crzp(text: str) -> str: + """CRZP answer text inside a combined answer: its own source lines go to the common "Zdroje CRZP" line.""" + lines = [l for l in text.split("\n") if not re.match(r"^\s*(?:-\s*)?Zdroj:\s", l)] + return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() + + +def explicit_part(question: str, result: dict) -> dict | None: + """A whole CRZP answer (CRZP is the main source), with its conversation token for follow-ups.""" + from tools.crzp.answering import answer as crzp, context as crzp_context + if result.get("status") not in ("ok", "ambiguous"): + return None + rows = result.pop("_shown", None) or [] + asked = result.pop("_asked", None) + options = result.pop("_options", None) + token = None + if asked is not None: + issued = crzp_context.issue(question, asked, rows, options) + token = issued["token"] if issued else None + return {"mode": "explicit", "status": result["status"], "text": clean_crzp(result["answer"]), "rows": rows, + "urls": result.get("source_urls", []), "crzp_token": token, + "people": [(_names(r.get("authors", [])), crzp.university(r)) for r in rows]} + + +def _kind(row) -> str: + from tools.crzp.answering.answer import _kind as kind + return kind(row) + + +def _names(values) -> str: + from tools.crzp.source.parse import display_name + return ", ".join(display_name(v) for v in values) or "neuvedené" + + +def _work(row) -> str: + return f"{_kind(row)} {row.get('year') or ''}".strip() + f" „{row['title']}“" + + +def crzp_methods(person: str, row: dict, detail: dict | None) -> str: + """"Ján Pták vo svojej bakalárskej práci „…“ (2025) využíva: ESP32, MQTT, InfluxDB, Home Assistant." — + the work's keywords and the tools its abstract names.""" + from tools.answers.content_selection import named_tools + record = detail or row + abstract = record.get("abstract") or "" + found = [] + for keyword in record.get("keywords", []): + spelled = re.search(r"(?i)(? 1 and not any(f" {key} " in f" {normalized(t)} " for t in tools): + tools = [t for t in tools if f" {normalized(t)} " not in f" {key} "] + tools.append(name) + where = {"bachelor_thesis": "vo svojej bakalárskej práci", "diploma_thesis": "vo svojej diplomovej práci", + "dissertation": "vo svojej dizertačnej práci"}.get(row.get("work_type"), "vo svojej práci") + return f"{person} {where} „{row['title']}“ ({row.get('year')}) využíva: {', '.join(tools[:12])}." + + +def remember(rows: list[dict]) -> None: + """Project the used CRZP works into Neo4j, linked to ZP Wiki people and topics (in the background).""" + from tools.crzp.storage import graph + rows = [r for r in rows if r.get("sid")] + if rows and os.getenv("CRZP_GRAPH_LINKS", "1") != "0": + _pool().submit(graph.project, rows) + + +def crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None: + """The CRZP contribution: {"mode", "text", "rows", "people", "works"} or None.""" + part = _crzp_part(question, person, complete=complete, student=student) + if part: + remember(part.get("rows") or []) + return part + + +def _crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None: + from tools.crzp.answering import answer as crzp + from tools.crzp.source.client import CrzpUnavailable + try: + planned = crzp_plan(question, person, complete, student) + if planned is None: + return None + mode, query = planned + low = normalized(question) + if mode == "similar": + return crzp_similar(person, complete=complete) + if mode == "explicit": + return explicit_part(question, crzp._answer_query(question, query, complete)) + found, rows = crzp.find(query) + if not rows: + return None + rows = crzp.ranked(question, query, rows) + if mode == "topic": + rows = rows[:5] + lines = [f"- {_names(r.get('authors', []))} — {crzp.university(r)}: „{r['title']}“ ({_kind(r)}, {r.get('year')})" + for r in rows] + return {"mode": mode, "text": "Práce na túto tému v CRZP (najbližšie podľa obsahu):\n\n" + "\n".join(lines), + "rows": rows, "urls": [r["url"] for r in rows if r.get("url")], + "people": [(_names(r.get("authors", [])), crzp.university(r)) for r in rows]} + # person: that person's works at TUKE + rows = [r for r in rows if crzp.university(r) == TUKE] + if not rows: + return None + if not student: # a supervisor: the works he or she led, newest first + rows = sorted(rows, key=lambda r: -(r.get("year") or 0)) + lines = [f"- {_names(r.get('authors', []))} — „{r['title']}“ ({_kind(r)}, {r.get('year')})" for r in rows[:8]] + more = f"\n\nSpolu {len(rows)} prác v CRZP na TUKE; zobrazených je 8 najnovších." if len(rows) > 8 else "" + return {"mode": mode, "text": f"Práce, ktoré {person} viedol alebo oponoval, v CRZP (TUKE):\n\n" + + "\n".join(lines) + more, "rows": rows[:8], "urls": [r["url"] for r in rows[:8] if r.get("url")], + "people": [(_names(r.get("authors", [])), crzp.university(r)) for r in rows[:8]]} + lines, details = [], {} + for row in rows[:3]: + if ABOUT.search(low) or METHODS.search(low): + details[row.get("sid")] = crzp.fetch.detail(row["sid"]) if row.get("sid") else None + detail = details.get(row.get("sid")) + if METHODS.search(low): + lines.append(crzp_methods(person, row, detail)) + elif ABOUT.search(low) and detail: + label, text = crzp.summary(detail, complete) + lines.append(f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{text}") + else: + lines.append(f"- {_work(row)} — vedúci {_names(row.get('supervisors', []))}; " + f"oponent {_names(row.get('opponents', []))}") + return {"mode": mode, "text": "\n\n".join(lines), "rows": rows[:3], + "urls": [r["url"] for r in rows[:3] if r.get("url")], "people": [], + "works": [{"source": "crzp", "label": _work(r), "type": r.get("work_type"), "key": crzp.work_key(r)} + for r in rows[:3]]} + except CrzpUnavailable: + return None + + +def crzp_similar(person: str, *, complete=None) -> dict | None: + """CRZP works sharing the person's specific tools/topics (searched in CRZP by them).""" + from tools.crzp import fetch + from tools.crzp.answering import answer as crzp + from tools.crzp.source.parse import fold + terms = wiki_terms(person) + if not terms: + return None + scored: dict = {} + for term in terms: + for row in fetch.search(term)["rows"]: + if any(fold(person.split()[-1]) in fold(a) for a in row.get("authors", [])): + continue + text = fold(row["title"] + " " + " ".join(row.get("keywords", []))) + shared = [t for t in terms if fold(t) in text] + if shared: + key = crzp.work_key(row) + scored[key] = (row, sorted(set(shared) | set(scored.get(key, (None, []))[1]))) + if not scored: + return None + candidates = list(scored.values()) + from tools.retrieval import rerank + profile = wiki_profile(person) + order = rerank.scores(profile, [r["title"] + ". " + ", ".join(r.get("keywords", [])) for r, _ in candidates]) \ + or [0] * len(candidates) + # Closest by content first; more shared topics/tools break ties. + best = [c for _, c in sorted(zip(order, candidates), key=lambda i: (-i[0], -len(i[1][1])))][:5] + lines = [f"- {_names(r.get('authors', []))} — {crzp.university(r)}: „{r['title']}“ ({_kind(r)}, {r.get('year')}); " + f"spoločné: {', '.join(shared)}" for r, shared in best] + return {"mode": "similar", "text": "Podobné práce v CRZP (spoločné témy a nástroje):\n\n" + "\n".join(lines), + "rows": [r for r, _ in best], "urls": [r["url"] for r, _ in best if r.get("url")], + "people": [(_names(r.get("authors", [])), crzp.university(r)) for r, _ in best]} + + +# ------------------------------------------------------------------ composition + +def _wiki_people(result: dict) -> list[tuple[str, str]]: + names = re.findall(r"^\*\*([^*\n—]+?)(?:\s+—[^*\n]*)?\*\*\s*$", result.get("answer") or "", re.M) + return [(n.strip(), WIKI_SCHOOL) for n in dict.fromkeys(names) if len(n.split()) >= 2] + + +def _schools_answer(question: str, engine: str, people: list) -> dict: + lines = [f"- {name} — {school}" for name, school in people] + text = "Školy študentov z predchádzajúcej odpovede:\n\n" + "\n".join(lines) + return _ok(question, engine, text, [], {}) + + +def _ok(question, engine, text, urls, base) -> dict: + keep = {k: v for k, v in base.items() if k not in ("answer", "status", "reason", "delivery_check", + "source_urls", "conversation_context")} + return {**keep, "status": "ok", "reason": None, "accepted_for_delivery": True, "answer": text, + "source_urls": list(dict.fromkeys(urls)), "claims": base.get("claims", []), + "index_version": base.get("index_version"), "complete": base.get("complete", True), + "truncated": base.get("truncated", False), "truncation_reasons": base.get("truncation_reasons", []), + "delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": fingerprint(question, engine)}} + + +def _with_token(result: dict, *, wiki_context=None, people=(), options=(), question="", crzp=None, + primary="wiki") -> dict: + wiki_context = wiki_context or {} + token = seal({"wiki": wiki_context.get("token"), "people": list(people)[:30], "options": list(options)[:6], + "question": question, "crzp": crzp, "primary": primary}) + if token: + result["conversation_context"] = {**{k: v for k, v in wiki_context.items() if k != "token"}, + "token": token, "work_options": wiki_context.get("work_options", []), + "meeting_options": wiki_context.get("meeting_options", [])} + return result + + +def answer(question: str, *, engine: str, limit: int = 5, context_token: str | None = None, complete=None, + deliver=None) -> dict: + """deliver: the ZP Wiki answer function (the API passes its own, tests replace it).""" + from tools.answers.answer_delivery import deliver_answer as default_deliver + from tools.answers.finalize import finalize + deliver_answer = deliver or default_deliver + from tools.crzp.answering.wiki_bridge import wiki_person + question = question.strip() + previous = unseal(context_token) + if previous and previous.get("error"): + from tools.answers.answer_errors import public_failure + failure = public_failure({"reason": previous["error"], "engine": engine}) + failure["delivery_check"] = {"layer": "context", "code": "unavailable", + "request_fingerprint": fingerprint(question, engine)} + return failure + wiki_token = previous.get("wiki") if previous else context_token + low = normalized(question) + if previous and previous.get("people") and SCHOOLS.search(low) and not wiki_person(question): + return _with_token(_schools_answer(question, engine, previous["people"]), people=previous["people"], + question=question) + if previous and previous.get("options"): + chosen = _choice(low, previous["options"]) + if chosen is not None: + return _answer_choice(previous["question"], chosen, engine=engine, limit=limit, complete=complete, + deliver=deliver_answer) + from tools.crzp.answering.wiki_bridge import wiki_match + match = wiki_match(question) + if previous and previous.get("primary") == "crzp" and previous.get("crzp") and not match: + # The last answer came from CRZP ("O čom je práca Jána Nováka?" -> "2", "a kto bol oponent?"). + from tools.crzp.answering import answer as crzp + followed = crzp._answer(question, complete, context_token=previous["crzp"]) + rows = followed.pop("_shown", None) or [] + followed.pop("_asked", None), followed.pop("_options", None) + if followed.get("accepted_for_delivery") is True: + urls = followed.get("source_urls") or [] + text = clean_crzp(followed["answer"]) + ("\n\nZdroje CRZP: " + ", ".join(urls[:8]) if urls else "") + result = {**followed, "answer": text, "delivery_check": {**followed["delivery_check"], + "request_fingerprint": fingerprint(question, engine)}} + token = (followed.get("conversation_context") or {}).get("token") + result.pop("conversation_context", None) + return _with_token(result, people=previous.get("people", []), question=question, + crzp=token or previous["crzp"], primary="crzp") + person, student = (match["name"], match["student"]) if match else (None, True) + started = time.monotonic() + pool = _pool() + try: + wiki_future = pool.submit(deliver_answer, question, engine=engine, limit=limit, + **({"context_token": wiki_token} if wiki_token else {})) + crzp_future = pool.submit(crzp_part, question, person, complete=complete, student=student) if enabled() else None + wiki = wiki_future.result() + try: + part = crzp_future.result(timeout=max(1.0, CRZP_SECONDS - (time.monotonic() - started))) if crzp_future else None + except FutureTimeout: + part = None # CRZP too slow this time: the wiki answer is not held back (its search stays cached) + except Exception: + part = None + finally: + pass # the shared pool stays; a slow CRZP part finishes in it and its search stays cached + if wiki.get("reason") in REFUSED or wiki.get("status") == "ambiguous" and ( + person or wiki.get("reason") not in WIKI_CLARIFICATION_YIELDS): + part = None # refused/out of scope, or the wiki asks first (which person, which work): no CRZP + wiki_ok = wiki.get("accepted_for_delivery") is True and wiki.get("status") == "ok" + if part is not None and part["mode"] == "explicit" and wiki.get("status") == "ambiguous": + wiki_ok = False + if wiki.get("accepted_for_delivery") is True and isinstance(wiki.get("answer"), str): + wiki = {**wiki, "answer": finalize(wiki["answer"], wiki.get("source_urls"))} + wiki_context = wiki.get("conversation_context") + if part is None or wiki.get("status") == "ambiguous" and part["mode"] != "explicit": + # (a wiki clarification such as "Uveď meno študenta" yields to a question CRZP answers as a whole) + people = _wiki_people(wiki) if wiki_ok else [] + return _with_token(wiki, wiki_context=wiki_context, people=people, question=question) if wiki.get( + "accepted_for_delivery") is True else wiki + # Different works in the two sources and the question names none: ask which one. + if person and part["mode"] == "person" and (METHODS.search(low) or ABOUT.search(low)): + options = _work_options(person, part, low) + if options: + text = (f"{person} má v podkladoch viac prác. Ktorú myslíš?\n\n" + + "\n".join(f"{i}. {o['label']} ({'ZP Wiki' if o['source'] == 'wiki' else 'CRZP'})" + for i, o in enumerate(options, 1)) + "\n\nOdpovedz číslom alebo typom práce.") + result = _ok(question, engine, text, [], wiki) + result.update(status="ambiguous", reason="clarification_required", + delivery_check={"layer": "planning", "code": "clarification_required", + "request_fingerprint": fingerprint(question, engine)}) + return _with_token(result, wiki_context=wiki_context, options=options, question=question) + if part["mode"] == "explicit" and part.get("status") == "ambiguous": # CRZP asks which namesake + result = _ok(question, engine, part["text"], [], wiki if wiki_ok else {"index_version": wiki.get("index_version")}) + result.update(status="ambiguous", reason="clarification_required", + delivery_check={"layer": "planning", "code": "clarification_required", + "request_fingerprint": fingerprint(question, engine)}) + return _with_token(result, question=question, crzp=part.get("crzp_token"), primary="crzp") + crzp_text = "**V CRZP** (Centrálny register záverečných prác):\n\n" + part["text"] + if not wiki_ok and (part["mode"] == "explicit" or wiki.get("status") == "error"): + text, urls = part["text"], part["urls"] + elif wiki_ok: + text = wiki["answer"] + "\n\n" + crzp_text + urls = [*(wiki.get("source_urls") or []), *part["urls"]] + else: + text, urls = "V ZP Wiki k tomu nie je záznam.\n\n" + crzp_text, part["urls"] + if part["urls"]: + text += "\n\nZdroje CRZP: " + ", ".join(part["urls"][:8]) + result = _ok(question, engine, text, urls, wiki if wiki_ok else {"index_version": wiki.get("index_version")}) + people = (_wiki_people(wiki) if wiki_ok else []) + list(part.get("people") or []) + return _with_token(result, wiki_context=wiki_context, people=people, question=question, + crzp=part.get("crzp_token"), primary="crzp" if part["mode"] == "explicit" and not wiki_ok else "wiki") + + +def _work_options(person: str, part: dict, low: str) -> list[dict]: + """The person's works when the two sources hold works of different types and the question names none.""" + if re.search(r"\b(?:bakalar\w*|diplom\w*|dizerta\w*|projekt\w*)\b", low): + return [] + wiki = wiki_works(person) + wiki_types = {w["type"] for w in wiki.values()} + thesis = {"diploma_thesis": "diplom", "bachelor_thesis": "bakalar"} + crzp_only = [w for w in part.get("works", []) if w["type"] in thesis and w["type"] not in wiki_types] + if not crzp_only: + return [] + wiki_options = [{"source": "wiki", "label": label + (f" „{w['topic']}“" if w.get("topic") else ""), "work": label} + for label, w in wiki.items() if w["type"] in thesis][:2] + return wiki_options + crzp_only if wiki_options else [] + + +def _choice(low: str, options: list[dict]) -> dict | None: + number = re.fullmatch(r"\s*(?:cislo\s+)?(\d)\s*[.)]?\s*", low) + if number and 1 <= int(number.group(1)) <= len(options): + return options[int(number.group(1)) - 1] + if CHOICE_CRZP.search(low): + return next((o for o in options if o["source"] == "crzp"), None) + if CHOICE_WIKI.search(low): + return next((o for o in options if o["source"] == "wiki"), None) + return None + + +def _answer_choice(question: str, chosen: dict, *, engine: str, limit: int, complete=None, deliver=None) -> dict: + from tools.answers.finalize import finalize + deliver_answer = deliver + from tools.crzp.answering.wiki_bridge import wiki_person + if chosen["source"] == "wiki": + adjective = {"diplom": "diplomovej", "bakal": "bakalárskej"}.get(normalized(chosen["work"])[:6].rstrip("ao"), "") + adjective = "diplomovej" if normalized(chosen["work"]).startswith("diplom") else \ + "bakalárskej" if normalized(chosen["work"]).startswith("bakal") else adjective + asked = re.sub(r"\b(svojej\s+|jeho\s+|jej\s+)?(práci|práce|prácu)\b", lambda m: (m.group(1) or "") + adjective + " " + m.group(2), + question, count=1) if adjective else question + if asked == question: + asked = question.rstrip(" ?.") + f" ({chosen['work']})?" + result = deliver_answer(asked, engine=engine, limit=limit) + if result.get("accepted_for_delivery") is True and isinstance(result.get("answer"), str): + result = {**result, "answer": finalize(result["answer"], result.get("source_urls"))} + result["delivery_check"] = {**(result.get("delivery_check") or {}), + "request_fingerprint": fingerprint(question, engine)} + return result + person = wiki_person(question) + part = crzp_part(question, person, complete=complete) + rows = [r for r in (part or {}).get("rows", []) if chosen.get("key") and _key_of(r) == chosen["key"]] + if not part or not rows: + return _ok(question, engine, "Vybranú prácu sa v CRZP nepodarilo znova načítať. Skús otázku zopakovať.", [], {}) + from tools.crzp.answering import answer as crzp + row = rows[0] + detail = crzp.fetch.detail(row["sid"]) if row.get("sid") else None + low = normalized(question) + if METHODS.search(low): + text = crzp_methods(person, row, detail) + else: + label, summary = crzp.summary(detail, complete) if detail else ("", "") + text = f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{summary}" + urls = [row["url"]] if row.get("url") else [] + return _ok(question, engine, text + ("\n\nZdroj: " + urls[0] if urls else ""), urls, {}) + + +def _key_of(row: dict) -> str: + from tools.crzp.answering import answer as crzp + return crzp.work_key(row) diff --git a/tools/answers/content_selection.py b/tools/answers/content_selection.py index c68d4be..de315a9 100644 --- a/tools/answers/content_selection.py +++ b/tools/answers/content_selection.py @@ -21,6 +21,25 @@ ADMIN = r"\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*\s+(?:text NOT_COMPLETED = r"\b(?:nie je|nie su|nebol\w*|este nie|zatial nie|nebude|nebudu|nefung\w*|neimplementov\w*|nevytvoren\w*|nevyhodnoten\w*)\b" +# Named tools in a record: acronyms (RAG, MCP), CamelCase (OpenWebUI, GraphRAG), names with digits +# (Neo4J, Fast120, me5mini), domains, and common lower-case tool names. +TOOL_NAMES = set("""python pytorch tensorflow keras docker kubernetes unsloth llamafactory qlora lora optuna whisper +bert slovakbert mistral llama qwen olmo gemma huggingface transformers spacy stanza prodigy rasa langchain llamaindex +elasticsearch opensearch neo4j cypher postgres postgresql sqlite mysql mongodb redis fastapi flask django react vue +angular nodejs javascript typescript java kotlin matlab opencv yolo onnx ollama openwebui telegram discord grafana +prometheus jupyter colab git gitea github gitlab slurm faiss qdrant chroma milvus pandas numpy scikit sklearn skills +skill streamlit gradio langgraph cogdb""".split()) +TOOL_PATTERN = re.compile(r"(? list[str]: + """Tool and technology names the record mentions (not dates, not plain words).""" + found = [m for m in TOOL_PATTERN.findall(text or "") if not re.fullmatch(r"\d+[.:]\d*", m) and len(m) >= 2] + found += [w for w in re.findall(r"[A-Za-z][A-Za-z0-9]+", text or "") if w.lower() in TOOL_NAMES] + return list(dict.fromkeys(found)) + + def has_qualification(text: str) -> bool: return bool( re.search( @@ -136,15 +155,21 @@ def facets_for(text: str, role: str, question: str, phase: str | None = None) -> r"\b(?:problem\w*|tazkost\w*|tazke|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*|nefung\w*|nepasuj\w*|nesulad\w*|nekompatibil\w*)\b", n ) and role not in {"assignment", "task"}: facets.append("problems") + tools = named_tools(text) if re.search( r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*|indexov\w*|databaz\w*|webov\w* rozhrani\w*|frontend\w*|backend\w*|vyskus\w*|sumarizac\w*)\b", n, - ): + ) or tools or re.search(SEARCH_METHODS, n): actual_use = actual_methods_requested(question) + # A status/result record naming a tool ("Sú Skills aj MCP server", "začíname pracovať na + # GraphRAG a Neo4J") states real use unless it is phrased as a plan or as not done. + stated_in_status = (role in {"progress", "result"} and bool(tools or re.search(SEARCH_METHODS, n)) + and not re.search(PLANNED, n)) actual_statement = not re.search(NOT_COMPLETED, n) and ( re.search(COMPLETED, n) or re.search(r'\bvyskus(?:any|ana|ane|al|ala|ali)\b',n) or re.search(r"\b(?:pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|vieme indexov\w*)\b", n) or (not re.search(PLANNED,n) and re.search(r'\b(?:frontend|backend)\s+\w+',n)) + or stated_in_status ) if not actual_use or (role not in {"assignment", "task"} and actual_statement): facets.append("methods") diff --git a/tools/answers/conversation.py b/tools/answers/conversation.py index 603867d..b0094f0 100644 --- a/tools/answers/conversation.py +++ b/tools/answers/conversation.py @@ -23,6 +23,19 @@ MAX_TOKEN = 16000 LIFETIME = 24 * 60 * 60 MEETING_REF = re.compile(r'\b(?:vtedy|(?:v|na|o|z|pri) (?:danom|tomto|tom|tomto istom) stretnuti)\b') WORK_REF = re.compile(r'\b(?:(?:tej|tu|t[uú]) (?:druhej|druhu|prvej|prvu)|(?:pri|o) druhej) (?:praci|pracu|projekte)\b') +# A pronoun points at the previous person ("Kto bol jeho vedúci?", "Aké výsledky dosiahla?" is no pronoun). +PRONOUN_REF = re.compile(r'\b(?:jeho|jej|ho|mu|nemu|nej|nom|onej|on|ona|tento student|tato studentka|ten student|' + r'ta studentka|toho studenta|tej studentky|ten isty|ta ista|daneho studenta|danej studentky|' + r'tohto studenta|tejto studentky)\b') +# A short question made only of these words asks about an aspect of the previous subject +# ("Kto bol vedúci?", "Aké technológie použil?"); any other content word makes it a new question. +ASPECT_WORDS = set("""kto bol bola boli je su bude co aky aka ake akej akom akych ktory ktora ktore kedy kde ako kolko +mal mala mali ma maju robil robila pracoval pracovala pouzil pouzila pouzival pouzivala dosiahol dosiahla urobil +urobila napisal napisala veduci veduceho veducim skolitel skolitela konzultant konzultanta oponent oponenta vysledky +vysledok vysledkoch technologie technologiu nastroje nastroj metody metodu ciel ciele tema temu temy nazov nazvy +praca prace pracu praci rok roku roky stretnutie stretnutia stretnuti posledne prve ulohy ulohu stav pokrok zadanie +plan klucove slova tagy jazyk jazyky data dataset model modely sa si mu jej jeho ho tam v vo na o z zo s so k a aj ale +uz este teraz potom tiez prosim daj ukaz vypis povedz""".split()) BOTH = re.compile(r'^(?:(?:daj|uved|ukaz|vypis|porovnaj|chcem|prosim)\s+)?(?:obe|oba|obidv\w*)(?:\s+(?:prace|projekty))?[.!?]?$') @@ -163,7 +176,9 @@ def resolve_question(question, *, token=None, catalog, index, version): explicit_date=bool(re.search(r'\b\d{1,2}\.\s*\d{1,2}\.\s*[12]\d{3}\b',query)) # An explicit date is stronger than a redundant word such as "vtedy". needs_context=meeting_ref and not explicit_date or bool(work_ref) or both - short_followup=bool(context and (explicit_date or re.match(r'^(?:a |ale |čo |co |aké |ake |kedy |čo sa |co sa )',n))) + aspect_only=bool(n) and len(n.split())<=7 and all(w in ASPECT_WORDS or w.isdigit() for w in n.split()) + short_followup=bool(context and (explicit_date or re.match(r'^(?:a |ale |čo |co |aké |ake |kedy |čo sa |co sa )',n) + or PRONOUN_REF.search(n) or aspect_only)) if needs_context and context is None: return ResolvedQuestion(query,clarification=( 'Ktoré stretnutie máš na mysli? Uveď študenta a dátum alebo presné označenie stretnutia.' if meeting_ref diff --git a/tools/answers/finalize.py b/tools/answers/finalize.py new file mode 100644 index 0000000..081f95d --- /dev/null +++ b/tools/answers/finalize.py @@ -0,0 +1,113 @@ +"""Last presentation pass over every accepted ZP Wiki answer (RAG and GraphRAG). + +It changes wording only, never facts: +- sources are written as full links, never as numbers ("[1]"); +- notices that explain the system instead of answering ("Vedúci je určený + podľa autora profilovej stránky…", "Profil obsahuje aj záznamy bez úplného + dátumu…", "Rok nedopĺňam…") are dropped; +- "this detail is not recorded" lines stay only when the answer has nothing + else; then one plain sentence says so; +- technical labels ("Názov / téma zo zdroja:") become plain words. +""" +from __future__ import annotations + +import re + +DROP = [ + r"Vedúci je určený podľa autora profilovej stránky[^\n]*", + r"Profil obsahuje aj záznamy bez úplného dátumu[^\n]*", + r"Rok nedopĺňam z roku práce\.?", + r"Údaje sú z indexovan[^\n]*", + r"Indexovan[éý][^\n]*", + r"Zobrazený je obmedzený výber[^\n]*", +] +PHASE_TAG = re.compile(r"\s+\((?:zaznamenaný výsledok|priebežný stav|plán|zadanie|podklad|navrhovaný cieľ|téma alebo názov|" + r"úloha označená ako splnená)\)(?=\s*$)", re.M) +# Students whose only match is an assignment/plan or reading: one line at the end instead of a block each. +SIDE = {"téma je v zadaní, téme alebo pláne práce; zaznamenaný výsledok k nej sa nenašiel.": + "Téma je len v zadaní alebo pláne, bez zaznamenaného výsledku", + "záznamy dokladajú len štúdium alebo poznámky k téme, nie vlastnú realizáciu.": + "Len štúdium alebo poznámky k téme"} + + +BULLET = re.compile(r"^- (?P[^;\n:]+?); (?P[^;:\n]+?)(?:; záznam (?P[^:\n]+?))?: (?P.*)$", re.M) +LABEL_PREFIX = re.compile(r"^(?:-\s+|\*(?:Názov|Meno)[^*]*\*:\s*|(?:Názov|Téma)\s*:\s*)+") + + +def _bullet(match: re.Match) -> str: + text = LABEL_PREFIX.sub("", match.group("text").strip()).strip() + if not text or text.startswith("#"): + return "" # a bare heading ("## Diplomový projekt 2") says nothing + if re.fullmatch(r"(?:Vedúci\w*:\s*)?(?:(?:Ing|Mgr|doc|prof|Bc|RNDr)\.\s*)*[A-ZÁ-Ž]\w+\s+[A-ZÁ-Ž]\w+(?:,\s*PhD\.?)?\.?", text): + return "" # only the supervisor's name, already the subject of the answer + date = (match.group("date") or "").replace("(rok neuvedený)", "").strip(" :()") + return f"- {match.group('work').strip()}" + (f" ({date})" if date else "") + f": {text}" + + +def _collapse_side_people(blocks: list[str]) -> list[str]: + groups, current = [], [] + for block in blocks: + if re.fullmatch(r"\*\*[^*\n]+\*\*", block) and current: + groups.append(current) + current = [] + current.append(block) + if current: + groups.append(current) + kept, side = [], {} + for group in groups: + head = re.fullmatch(r"\*\*([^*\n]+)\*\*", group[0]) + label = next((v for k, v in SIDE.items() for b in group if b.endswith(k)), None) + if head and label: + side.setdefault(label, []).append(head.group(1)) + else: + kept += group + return kept + [f"{label}: {', '.join(names)}." for label, names in side.items()] +MISSING = re.compile(r"(?:požadovaný údaj nie je v tomto zápise doložený|v podkladoch chýbajú doložené údaje|" + r"Požadované údaje k tomuto stretnutiu nie sú v dostupnom výbere doložené)", re.I) +RENAME = [ + (r"Názov / téma zo zdroja:\s*(?:\*Názov [^*]*\*:\s*)?", "Téma: "), + (r"Vedúci práce \(autor profilu študenta\):", "Vedúci práce:"), + (r"Konzultant práce \(zo zdroja\):", "Konzultant:"), +] +INLINE_CITE = re.compile(r"\s*(?:\[\d+\])+(?!\(|:)") +REFERENCE_LINE = re.compile(r"^\[\d+\]:\s*\S+\s*$", re.M) +SOURCES = re.compile(r"^Zdroj(?:e)?:\s*(.+)$", re.M) + + +def _links(line: str) -> list[str]: + urls = re.findall(r"\]\((https?://[^)\s]+)\)", line) or re.findall(r"https?://[^\s,)\]]+", line) + return list(dict.fromkeys(urls)) + + +def finalize(text: str, source_urls: list[str] | None = None) -> str: + if not isinstance(text, str) or not text.strip(): + return text + out = REFERENCE_LINE.sub("", text) + sources, label = [], None + for match in SOURCES.finditer(out): + sources += _links(match.group(1)) + label = label or match.group(0).split(":", 1)[0] + out = SOURCES.sub("", out) + out = INLINE_CITE.sub("", out) + for pattern in DROP: + out = re.sub(pattern, "", out) + for pattern, replacement in RENAME: + out = re.sub(pattern, replacement, out) + out = PHASE_TAG.sub("", out) + out = BULLET.sub(_bullet, out) + out = re.sub(r"^- Téma alebo názov doložený pri tej istej práci:\s*(?:\*[^*]*\*:\s*)?", "- Téma: ", out, flags=re.M) + out = re.sub(r"(?m)^(\s*- [^:\n]+:)\s+-\s+", r"\1 ", out) # "Diplomový projekt: - Rozpoznávanie…' + blocks = _collapse_side_people([b.strip() for b in re.split(r"\n\s*\n", out) if b.strip()]) + substantive = [b for b in blocks if not MISSING.search(b) and not re.fullmatch(r"\*\*[^*]+\*\*", b) + and not re.fullmatch(r"(?:\*\*[^*]+\*\*\s*)?(?:Posledný záznam[^\n]*|Zadanie a ciele:)", b)] + if substantive: + blocks = [b for b in blocks if not MISSING.search(b)] + blocks = [b for i, b in enumerate(blocks) # a heading left without its content + if not (re.fullmatch(r"[A-ZÁ-Ž][^\n]{0,40}:", b) and (i + 1 == len(blocks) or blocks[i + 1].startswith("**")))] + else: + heads = [b for b in blocks if re.fullmatch(r"\*\*[^*]+\*\*", b) or b.startswith("**")] + blocks = heads[:1] + ["ZP Wiki k tomu nemá záznam."] + sources = list(dict.fromkeys(sources)) # rewrite the answer's own source line, never add one + if sources: + blocks.append((label or "Zdroje") + ": " + ", ".join(sources)) + return "\n\n".join(blocks) diff --git a/tools/answers/question_intent.py b/tools/answers/question_intent.py index 8f2aad5..92acd7a 100644 --- a/tools/answers/question_intent.py +++ b/tools/answers/question_intent.py @@ -7,7 +7,7 @@ FACET_PATTERNS = { "assignment": r"\b(?:zadani\w*|ciel\w*)\b|\bco\s+(?:ma|maju|mal[aoiy]?)\b[^?;!.]*\b(?:vytvorit|vypracovat|implementovat|navrhnut|zostavit)\b", "plans": r"\b(?:planovan\w*|dalsi\w*\s+uloh\w*|uloh\w*|co\s+(?:ma|maju|mal[aoiy]?)\s+(?:urobit|spravit)(?:\s+dalej)?)\b", "problems": r"\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b", - "methods": r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*|pristup\w*|frontend\w*|backend\w*)\b", + "methods": r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*|vyuziv\w*|vyuzil\w*|pristup\w*|frontend\w*|backend\w*)\b", "experiments": r"\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*|vyhodnot\w*)\b", "progress": r"\b(?:priebezn\w*\s+)?stav\w*\b", "results": r"\b(?:vysledk\w*|dosiahn\w*|funkcnost\w*)\b", @@ -144,7 +144,7 @@ def facet_mode(question: str) -> bool: def actual_methods_requested(question: str) -> bool: return bool( re.search( - r"\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*|vyskus(?:al|ala|ali)\w*)\b|" + r"\b(?:skutocne|pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|implementoval\w*|vyskus(?:al|ala|ali)\w*)\b|" r"\bake technolog\w* ma\b|" r"\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b", normalized(question), diff --git a/tools/answers/question_plan.py b/tools/answers/question_plan.py index e566f17..ea79a6c 100644 --- a/tools/answers/question_plan.py +++ b/tools/answers/question_plan.py @@ -174,7 +174,10 @@ def direct_spec(question, resolution, *, meeting=None, route=None, relative_year meeting=meeting.model_copy(update={'count':count}) facets=requested_facets(q) if route is not None and route.facets: - facets=list(route.facets) + # A meeting question asks for the whole meeting ("čo na ňom riešil") unless it names an + # aspect itself; the classifier must not narrow it to e.g. results it guessed. + # Aspects the question names itself ("aké úlohy dostal" -> plans) win over the classifier's guess. + facets=[f for f in route.facets if f in facets] if route.type=='meeting' else (facets or list(route.facets)) global_discovery=discovery_question(q) if route is None and mode not in {'similar','supervisor','advisors','completion','meeting','period','proposal'} and global_discovery: mode='discovery' diff --git a/tools/answers/synthesis.py b/tools/answers/synthesis.py index 037111a..36d1a12 100644 --- a/tools/answers/synthesis.py +++ b/tools/answers/synthesis.py @@ -489,9 +489,12 @@ def render(draft, rows, result, plan): bullets=bool(re.search(r'\bbod\w*\b',normalized(plan.question))) or plan.presentation=='list' and plan.intent!='meeting' text=('\n' if bullets else ' ').join(dict.fromkeys(('- ' if bullets else '')+c.text+' '+cites(c) for c in claims)) dates=' (dátum záznamu: '+displayed+')' if displayed and plan.intent!='meeting' else '' - title=({'result':'Výsledky','progress':'Priebeh a zaznamenané ťažkosti', - 'assignment':'Zadanie','planned':'Ďalší postup','unknown':'Doplňujúce podklady'}.get(phase,PHASES[phase]) + title=({'result':'Výsledky','progress':'Stav', + 'assignment':'Zadanie','planned':'Úlohy','unknown':'Ďalšie poznámky'}.get(phase,PHASES[phase]) if plan.intent=='meeting' else PHASES[phase]) + if phase=='planned' and list(plan.facets)==['plans']: # "A aké úlohy dostal?" + title='Úlohy'+(' zo stretnutia '+displayed if displayed else '');dates='';bullets=True + text='\n'.join(dict.fromkeys('- '+c.text+' '+cites(c) for c in claims)) sections.append(title+dates+(':\n' if bullets else ': ')+text) cell_dates=' (dátum záznamu: '+displayed+')' if displayed else '' table_cells[(scope,label,phase)].append((cell_dates+': ' if cell_dates else '')+text) diff --git a/tools/claude/build_claude_packages.py b/tools/claude/build_claude_packages.py index 3574e69..1637ea0 100644 --- a/tools/claude/build_claude_packages.py +++ b/tools/claude/build_claude_packages.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 -"""Build reproducible, self-contained Claude skill ZIPs without runtime data or secrets.""" +"""Build the self-contained Claude skills without runtime data or secrets. + +The skills live unpacked in packages/claude/dist// (SKILL.md, scripts, +manifest); no archive is kept in the repository. A reproducible ZIP for upload +to claude.ai is made only on request: + + python tools/claude/build_claude_packages.py # refresh dist// + python tools/claude/build_claude_packages.py --zip /tmp/x # also /tmp/x/zpwiki-.zip +""" from __future__ import annotations @@ -11,18 +19,18 @@ import zipfile ROOT = Path(__file__).resolve().parents[2] BASE = ROOT / "packages/claude" -# Skill sources live unpacked next to their archives: dist// holds -# exactly the files of dist/zpwiki-.zip (SKILL.md, scripts, manifest). +# Skill sources live unpacked: dist// (SKILL.md, scripts, manifest). SOURCES = BASE / "dist" -ENGINES = ("rag", "graphrag", "crzp") -PACKAGE_VERSION = "1.13.0" +ENGINES = ("rag", "graphrag") +PACKAGE_VERSION = "2.0.0" def json_bytes(value): return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode() -def build_package(engine: str, output: Path) -> dict: +def build_package(engine: str, output: Path, archive: Path | None = None) -> dict: + """Write output//; with archive, also archive/zpwiki-.zip.""" if engine not in ENGINES: raise ValueError("Unknown engine") name = engine @@ -49,36 +57,38 @@ def build_package(engine: str, output: Path) -> dict: } ) output.mkdir(parents=True, exist_ok=True) - # The unpacked folder is kept identical to the archive (shared client.py - # and the manifest are refreshed from their single source on every build). + # Shared client.py and the manifest are refreshed from their single source on every build. folder = output / name for path, data in files.items(): target = folder / path target.parent.mkdir(parents=True, exist_ok=True) if not target.exists() or target.read_bytes() != data: target.write_bytes(data) - destination = output / ("zpwiki-" + name + ".zip") - with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as archive: + result = {"name": name, "folder": name, + "manifest_sha256": hashlib.sha256(files["manifest.json"]).hexdigest()} + if archive is None: + return result + archive.mkdir(parents=True, exist_ok=True) + destination = archive / ("zpwiki-" + name + ".zip") + with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as bundle: for path, data in sorted(files.items()): info = zipfile.ZipInfo(name + "/" + path, date_time=(1980, 1, 1, 0, 0, 0)) info.compress_type = zipfile.ZIP_DEFLATED info.create_system = 3 info.external_attr = 0o100644 << 16 - archive.writestr(info, data) - return { - "name": name, - "archive": destination.name, - "sha256": hashlib.sha256(destination.read_bytes()).hexdigest(), - } + bundle.writestr(info, data) + return {**result, "archive": destination.name, "sha256": hashlib.sha256(destination.read_bytes()).hexdigest()} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, default=BASE / "dist") + parser.add_argument("--zip", type=Path, help="also write zpwiki-.zip for upload into this folder") args = parser.parse_args() - packages = [build_package(engine, args.output) for engine in ENGINES] + packages = [build_package(engine, args.output, args.zip) for engine in ENGINES] (args.output / "packages.json").write_bytes( - json_bytes({"schema_version": 1, "packages": packages}) + json_bytes({"schema_version": 2, "packages": [{k: p[k] for k in ("name", "folder", "manifest_sha256")} + for p in packages]}) ) print(json.dumps(packages, ensure_ascii=False)) diff --git a/tools/crzp/answering/answer.py b/tools/crzp/answering/answer.py index d5170dc..6cf0374 100644 --- a/tools/crzp/answering/answer.py +++ b/tools/crzp/answering/answer.py @@ -61,6 +61,8 @@ def keep(row: dict, query: CrzpQuery) -> bool: return False # follow-up about chosen works of the previous answer if query.school_name and university(row) != query.school_name: return False # the chosen namesake + if query.works and work_key(row) not in query.works: + return False # the chosen namesake's works (two students of one name at one university) if any(not people_in_role(row, p.name, p.role) for p in query.people): return False if query.years and row.get("year") not in query.years: @@ -147,20 +149,70 @@ def prefer_exact(rows: list[dict], query: CrzpQuery) -> list[dict]: return rows +LEVEL = {"bachelor_thesis": 1, "diploma_thesis": 2, "doctoral_thesis": 3, "dissertation": 3, "habilitation_thesis": 4} + + +def work_key(row: dict) -> str: + return f"{' '.join(fold(row['title']).rstrip(' .').split())}|{row.get('year')}" + + +def _split_students(group: dict) -> list[dict]: + """One student writes one work of each kind: two bachelor theses of "Ján Novák" at one + university are two persons. Works are given, oldest first, to the first person who + has no work of that kind yet and whose last work is at most 8 years older.""" + persons = [] + for row in sorted(group["rows"], key=lambda r: r.get("year") or 0): + kind, year = row.get("work_type"), row.get("year") or 0 + target = next((p for p in persons if kind not in p["kinds"] and year - p["last"] <= 8), None) + if target is None: + target = {"kinds": set(), "last": year, "rows": []} + persons.append(target) + target["kinds"].add(kind) + target["last"] = max(target["last"], year) + target["rows"].append(row) + return [{**group, "rows": p["rows"], "count": len(p["rows"]), "years": {r.get("year") for r in p["rows"]}} + for p in persons] + + def identities(rows: list[dict], person) -> list[dict]: - """The asked name split into probable persons: the same written name (with diacritics) - at the same university is one person; another spelling or university is another one. - CRZP has no person identifiers, so this is the most it can tell.""" + """The asked name split into probable persons (CRZP has no person identifiers): + - the same written name (with diacritics) at the same university is one person, + another spelling or university another one; + - a name that is only an author there is split when it has two works of one kind + (two bachelor theses are two students); + - an author at another university whose earlier work leads to this one + (bachelor -> diploma within 4 years) is marked as possibly the same person.""" groups: dict = {} + seen = set() for row in rows: + if work_key(row) in seen: + continue # the same work held twice in CRZP + seen.add(work_key(row)) for name in people_in_role(row, person.name, person.role): key = (display_name(name), university(row)) group = groups.setdefault(key, {"name": key[0], "school": key[1], "count": 0, "years": set(), - "roles": Counter()}) + "roles": Counter(), "rows": []}) group["count"] += 1 group["years"].add(row.get("year")) + group["rows"].append(row) group["roles"].update(role for role, field in ROLE_FIELD.items() if name in row.get(field, [])) - return sorted(groups.values(), key=lambda g: (-g["count"], -max((y or 0) for y in g["years"]), g["name"], g["school"])) + result = [] + for group in groups.values(): + result += _split_students(group) if set(group["roles"]) == {"author"} else [group] + for group in result: + group["hint"] = None + if set(group["roles"]) != {"author"}: + continue + first = min(group["rows"], key=lambda r: r.get("year") or 0) + for other in result: + if other is group or other["name"] != group["name"] or set(other["roles"]) != {"author"}: + continue + last = max(other["rows"], key=lambda r: r.get("year") or 0) + if (other["school"] != group["school"] + and LEVEL.get(last.get("work_type"), 0) < LEVEL.get(first.get("work_type"), 0) + and 0 < (first.get("year") or 0) - (last.get("year") or 0) <= 4): + group["hint"] = f"možno tá istá osoba ako {other['school']} ({last.get('year')})" + return sorted(result, key=lambda g: (-g["count"], -max((y or 0) for y in g["years"]), g["name"], g["school"])) def _identity_line(group: dict) -> str: @@ -168,7 +220,8 @@ def _identity_line(group: dict) -> str: span = (f"{years[0]}–{years[-1]}" if len(years) > 1 else str(years[0])) if years else "rok neuvedený" roles = ", ".join(ROLE_LABEL[r] for r, _ in group["roles"].most_common()) works = f"{group['count']} {'práca' if group['count'] == 1 else 'práce' if group['count'] < 5 else 'prác'}" - return f"{group['name']} — {group['school']} ({roles}; {span}; {works})" + hint = f"; {group['hint']}" if group.get("hint") else "" + return f"{group['name']} — {group['school']} ({roles}; {span}; {works}{hint})" def namesakes(rows: list[dict], query: CrzpQuery) -> str: @@ -185,7 +238,7 @@ def namesakes(rows: list[dict], query: CrzpQuery) -> str: def clarification(rows: list[dict], query: CrzpQuery) -> tuple[str, list[dict]] | None: """Count, summary, keywords, relations and similar works of one person must not mix namesakes: ask which person is meant (a list shows all with a notice instead).""" - if not query.people or query.intent in ("list", "schools") or query.school_name or query.sids: + if not query.people or query.intent in ("list", "schools") or query.school_name or query.sids or query.works: return None groups = identities(rows, query.people[0]) if len(groups) < 2: @@ -195,21 +248,25 @@ def clarification(rows: list[dict], query: CrzpQuery) -> tuple[str, list[dict]] "identifikátorom). Ktorú myslíš?\n\n" + "\n".join(f"{i}. {_identity_line(g)}" for i, g in enumerate(groups, 1)) + "\n\nOdpovedz číslom, školou alebo celým menom.") - return text, [{"name": g["name"], "school": g["school"]} for g in groups] + return text, [{"name": g["name"], "school": g["school"], "works": [work_key(r) for r in g["rows"]][:30]} + for g in groups] def _merge(results: list[dict]) -> dict: - rows, seen = [], set() + rows, seen = [], {} for result in results: for row in result["rows"]: - key = row.get("sid") or (fold(row["title"]), row.get("year"), tuple(row.get("authors", []))) + key = (fold(row["title"]).rstrip(" ."), row.get("year"), tuple(row.get("authors", []))) if key not in seen: - seen.add(key) + seen[key] = row rows.append(row) + elif row.get("sid") and not seen[key].get("sid"): # the same work read with its permalink + seen[key].update(sid=row["sid"], url=row.get("url")) return {"texts": [r["query"] for r in results], "totals": [r["total"] for r in results], "total": results[0]["total"] if len(results) == 1 else sum(r["total"] for r in results), "facets": results[0]["facets"], "rows": rows, "complete": all(r["complete"] for r in results), - "incomplete": [(r["query"], r["total"], len(r["rows"])) for r in results if not r["complete"]]} + "incomplete": [(r["query"], r["total"], len(r["rows"])) for r in results if not r["complete"]], + "parts": results} def _unverified(query: CrzpQuery, text: str) -> str | None: @@ -229,22 +286,48 @@ def _used_with_topic(term: str, rows: list[dict]) -> bool: AGGREGATES = ("count", "keywords", "schools", "relation") -def find(query: CrzpQuery, *, links: bool = True) -> tuple[dict | None, list[dict]]: +def by_facets(query: CrzpQuery) -> bool: + """A count, or the supervisors of works on a topic, that CRZP computes itself: every + condition is in the search (text, years, work type) and one text per meaning, so the + CRZP total and the full year/type/supervisor facets are exact without reading rows.""" + counted = query.intent == "count" or query.intent == "relation" and query.target_role == "supervisor" + return (counted and not query.people and not query.title and not query.sids and not query.school_name + and not query.alternatives and len(query.topics) <= 1) + + +def reader(query: CrzpQuery): + """How one search text is read for this question: + - counted by CRZP (by_facets): the first page with the full facets only; + - summaries (tags, schools, opponents, a person's works): every row up to + DEEP_ROWS, in year ranges at once, permalinks only where a work is shown; + - a list of a school or workplace: the newest LINKED_ROWS works, all with permalinks; + - a topic list, a summary of a work, similar works: the LINKED_ROWS most relevant rows.""" + window = filters(query) + if by_facets(query): + return lambda text: fetch.search(text, links=False, depth=20, **window) + if query.intent in ("keywords", "schools", "count"): + return lambda text: fetch.search_deep(text, links=False, **window) + if query.intent == "relation" or query.people: + return lambda text: fetch.search_deep(text, links=True, **window) + if query.intent == "list" and (query.school or query.unit) and not query.topics and not query.title: + return lambda text: fetch.search_deep(text, links=True, limit=fetch.LINKED_ROWS, **window) + return lambda text: fetch.search(text, links=query.intent != "count", **window) + + +def find(query: CrzpQuery) -> tuple[dict | None, list[dict]]: """First search group whose rows satisfy all conditions; (merged result, rows). A model-proposed alternative term is searched only if CRZP uses it in a work already found for the topic, or if nothing was found yet.""" last, seen = None, [] query.searched = [] - # Counts and summaries read up to DEEP_ROWS rows (no permalinks needed); a person's - # works are filtered by role in code, so all of them are read as well. - depth = fetch.DEEP_ROWS if query.intent in AGGREGATES or query.people else fetch.LINKED_ROWS + read = reader(query) for group in search_groups(query): results, verified_rows, unverified = [], 0, [] for text in group: term = _unverified(query, text) if term and seen and not _used_with_topic(term, seen): continue # "rozložená kniha" for blockchain: CRZP does not use it, no search - results.append(fetch.search(text, links=links, depth=depth, **filters(query))) + results.append(read(text)) query.searched.append(text) seen += results[-1]["rows"] if term: @@ -255,6 +338,12 @@ def find(query: CrzpQuery, *, links: bool = True) -> tuple[dict | None, list[dic continue found = _merge(results) rows = prefer_exact([r for r in found["rows"] if keep(r, query)], query) + if by_facets(query) and not found["complete"]: + if len(rows) == len(found["rows"]): + found["facet_only"] = True # CRZP itself counted it; every read row meets the conditions + else: # some found works are elsewhere (e.g. a school name in a title): count the rows themselves + found = _merge([fetch.search_deep(r["query"], links=False, **filters(query)) for r in found["parts"]]) + rows = prefer_exact([r for r in found["rows"] if keep(r, query)], query) last = (found, rows) if rows and unverified and not verified_rows: query.unverified_only = unverified # the topic itself found nothing: say so in the answer @@ -326,8 +415,8 @@ def work_block(row: dict, detail: dict | None = None) -> str: def work_line(row: dict) -> str: - return (f"- **{row['title']}** ({_kind(row)}, {row.get('year') or '?'}, {school(row)}) — " - f"{_names(row.get('authors', []))} · {_link(row)}") + return (f"- {_names(row.get('authors', []))} — „{row['title']}“ ({_kind(row)}, {row.get('year') or '?'}, " + f"{school(row)}) · {_link(row)}") def describe(query: CrzpQuery) -> str: @@ -354,10 +443,13 @@ def describe(query: CrzpQuery) -> str: # ------------------------------------------------------------------ model steps -SUMMARY_PROMPT = ("Na základe abstraktu a kľúčových slov záverečnej práce napíš po slovensky 2 až 3 vety: o čom práca " - "je a čo v nej autor urobil. Môžeš preformulovať, ale použi iba informácie z abstraktu; nepridávaj " - "výsledky, nástroje ani hodnotenia, ktoré v ňom nie sú. Bez úvodu a bez nadpisu.") -CHECK_PROMPT = ("Over, či je každé tvrdenie zhrnutia podložené abstraktom (preformulovanie je v poriadku, nový fakt nie). " +SUMMARY_PROMPT = ("Si odborník, ktorý stručne vysvetľuje záverečné práce. Z abstraktu a kľúčových slov napíš po " + "slovensky 2 až 3 vlastné vety: o čom práca je, čo v nej autor urobil a s akými nástrojmi alebo " + "metódami. Píš vlastnými slovami a súvislo, neopisuj vety abstraktu doslova a nezačínaj slovami " + "„Táto práca“ ani „Abstrakt“. Použi iba informácie z abstraktu a kľúčových slov; nepridávaj " + "výsledky, čísla, nástroje ani hodnotenia, ktoré v nich nie sú. Bez nadpisu a bez odrážok.") +CHECK_PROMPT = ("Over, či je každé tvrdenie zhrnutia podložené abstraktom alebo kľúčovými slovami (preformulovanie " + "a zovšeobecnenie je v poriadku, nový fakt, číslo alebo nástroj nie). " "Odpovedz iba JSON {\"supported\": true} alebo {\"supported\": false}.") @@ -373,17 +465,30 @@ def _ask(system: str, payload: dict, complete=None, max_tokens: int = 400) -> st def summary(detail: dict, complete=None) -> tuple[str, str]: - """("Zhrnutie", text) after a passed check, else ("Abstrakt", abstract).""" + """("Zhrnutie", text): the model's own 2-3 sentences after a passed check (second try with + the check's verdict), else a sentence built from the CRZP keywords — never the copied abstract.""" abstract = detail.get("abstract") or detail.get("abstract_secondary") or "" + keywords = detail.get("keywords", []) if not abstract: - return "Abstrakt", "CRZP pri tejto práci abstrakt neuvádza." - text = _ask(SUMMARY_PROMPT, {"title": detail["title"], "abstract": abstract, "keywords": detail.get("keywords", [])}, - complete) - if text and len(text) < 1200: - verdict = _ask(CHECK_PROMPT, {"abstract": abstract, "summary": text}, complete, max_tokens=50) + return "Zhrnutie", _keyword_sentence(detail) if keywords else "CRZP pri tejto práci abstrakt neuvádza." + payload = {"title": detail["title"], "abstract": abstract, "keywords": keywords} + for attempt in range(2): + text = _ask(SUMMARY_PROMPT if attempt == 0 else SUMMARY_PROMPT + " Drž sa výlučne faktov z abstraktu.", + payload, complete) + if not text or len(text) > 1200: + continue + verdict = _ask(CHECK_PROMPT, {"abstract": abstract, "keywords": keywords, "summary": text}, complete, max_tokens=50) if verdict and re.search(r'"supported"\s*:\s*true', verdict): - return "Zhrnutie podľa abstraktu", text - return "Abstrakt", abstract if len(abstract) <= 1500 else abstract[:1500].rsplit(" ", 1)[0] + " …" + return "Zhrnutie", text + return "Zhrnutie", _keyword_sentence(detail) + + +def _keyword_sentence(detail: dict) -> str: + keywords = [k for k in detail.get("keywords", []) if len(k) < 40][:6] + kind = WORK_TYPE_LABELS.get(detail.get("work_type"), "práca").capitalize() + if not keywords: + return f"{kind} sa podľa CRZP zaoberá témou „{detail['title']}“." + return f"{kind} „{detail['title']}“ sa zaoberá témami: {', '.join(keywords)}." # ------------------------------------------------------------------ answers by intent @@ -391,6 +496,10 @@ def summary(detail: dict, complete=None) -> tuple[str, str]: def _list(question, query, found, rows) -> tuple[str, list[dict]]: rows = ranked(question, query, rows) header = f"V CRZP som našiel {len(rows)} {'prácu' if len(rows) == 1 else 'práce' if len(rows) < 5 else 'prác'} ({describe(query)}):" + if not found["complete"] and found["total"] > len(found["rows"]): + newest = not query.topics and not query.title and not query.people + header = (f"V CRZP je {found['total']} výsledkov hľadania ({describe(query)}); prehľadaných bolo " + f"{len(found['rows'])} {'najnovších' if newest else 'najrelevantnejších'}, z nich zodpovedá {len(rows)}:") if len(rows) <= 5: return header + "\n\n" + "\n\n".join(work_block(r) for r in rows), rows shown = rows[:LIST_LIMIT] @@ -400,28 +509,35 @@ def _list(question, query, found, rows) -> tuple[str, list[dict]]: return text, shown +TYPE_FACET = {"bakalárska": "bakalárska práca", "diplomová": "diplomová práca", "dizertačná": "dizertačná práca", + "habilitačná": "habilitačná práca", "rigorózna": "rigorózna práca"} + + +def _facet_type(label: str) -> str: + return next((v for k, v in TYPE_FACET.items() if k in label.lower()), label) + + def _count(question, query, found, rows) -> tuple[str, list[dict]]: count, note = len(rows), "" person = query.people[0] if query.people else None - if not found["complete"]: - facet = FACET.get(person.role) if person else None - values = [f["count"] for f in found["facets"].get(facet, []) if name_matches(person.name, f["label"])] if facet else [] - # The facet counts the whole CRZP result, which already respects the year and type filters. - school_only = (query.school and not query.people and not query.topics and not query.title - and found["texts"] == (UNITS[query.unit]["search"] if query.unit else [SCHOOLS[query.school][0]])) - if values and len(found["texts"]) == 1 and not (query.school or query.unit or len(query.people) > 1): - count, note = sum(values), " (súhrnný počet z CRZP)" - elif school_only and len(rows) == len(found["rows"]): - count = found["total"] - several = len(found["texts"]) > 1 - note = (f" (súhrn CRZP pre hľadanie {'pracoviska (kód a starší zápis názvom)' if several else 'kódu pracoviska' if query.unit else 'názvu školy'} " - f"s filtrom roku a typu práce; všetkých {len(rows)} prehľadaných záznamov bolo z tohto pracoviska)") - else: - note = f" (aspoň; prehľadaných {len(found['rows'])} z {found['total']} výsledkov hľadania)" types = Counter(_kind(r) for r in rows) years = Counter(r.get("year") for r in rows if r.get("year")) + if found.get("facet_only"): + # CRZP's own total and full facets (texts of one workplace are disjoint: code vs. old written name) + count, types, years = found["total"], Counter(), Counter() + for part in found["parts"]: + years.update(fetch.year_counts(part)) + types.update({_facet_type(f["label"]): f["count"] for f in part["facets"].get("Typ práce", [])}) + note = " (súhrnný počet z CRZP)" + elif not found["complete"]: + facet = FACET.get(person.role) if person else None + values = [f["count"] for f in found["facets"].get(facet, []) if name_matches(person.name, f["label"])] if facet else [] + if values and len(found["texts"]) == 1 and not (query.school or query.unit or len(query.people) > 1): + count, note = sum(values), " (súhrnný počet z CRZP)" + else: + note = f" (aspoň; prehľadaných {len(found['rows'])} z {found['total']} výsledkov hľadania)" text = f"Počet prác v CRZP ({describe(query)}): **{count}**{note}." - if rows: + if count: text += "\n\n- Podľa typu: " + ", ".join(f"{k} {v}" for k, v in types.most_common()) text += "\n- Podľa roku: " + ", ".join(f"{y}: {n}" for y, n in sorted(years.items(), reverse=True)) return text, rows @@ -443,11 +559,15 @@ def _about(question, query, found, rows, complete=None) -> tuple[str, list[dict] summaries = list(pool.map(lambda d: summary(d, complete) if d else None, details)) blocks = [] for row, detail, done in zip(rows, details, summaries): + head = (f"**{row['title']}** — {_kind(row)}, {row.get('year') or 'rok neuvedený'}, {university(row)}; " + f"autor {_names(row.get('authors', []))}") + if re.search(r"oponent|ved[uú]c|[sš]kolite", question.lower()): # "Kto bol oponentom … a o čom je?" + head += f"; vedúci {_names(row.get('supervisors', []))}; oponent {_names(row.get('opponents', []))}" if detail is None: - blocks.append(work_block(row) + "\n\nDetail záznamu (abstrakt) nie je dostupný bez trvalého odkazu.") + blocks.append(head + "\n\nCRZP k tejto práci neposkytuje trvalý odkaz, preto abstrakt nemám.") continue - label, text = done - blocks.append(work_block(row, detail) + f"\n\n{label}: {text}") + _, text = done + blocks.append(head + f"\n\n{text}\n\nZdroj: {_link(row)}") more = len(candidates) > len(rows) tail = "\n\nZodpovedá viac prác; zobrazené sú najrelevantnejšie. Spresni typ práce alebo rok." if more else "" return "\n\n".join(blocks) + tail, rows @@ -478,6 +598,15 @@ def _relation(question, query, found, rows) -> tuple[str, list[dict]]: plural = lambda c: f"{c} {'práca' if c == 1 else 'práce' if c < 5 else 'prác'}" if not query.people: # "Kto viedol práce o X?": the target role over the found works counts = Counter(n for r in rows for n in r.get(ROLE_FIELD[target], [])) + if (found.get("facet_only") and target == "supervisor" and not found["complete"] + and all(part["facets"].get("Školiteľ") for part in found["parts"])): # CRZP's full supervisor facet + counts = Counter() + for part in found["parts"]: + counts.update({f["label"]: f["count"] for f in part["facets"].get("Školiteľ", [])}) + total = found["total"] + lines = [f"- {display_name(n)} ({plural(c)})" for n, c in sorted(counts.items(), key=lambda i: (-i[1], i[0]))[:15]] + return (f"{ROLE_PLURAL[target]} prác ({describe(query)}; {total} prác v CRZP, súhrn CRZP):\n\n" + + ("\n".join(lines) or "- CRZP ich neuvádza")), ranked(question, query, rows)[:10] lines = [f"- {display_name(n)} ({plural(c)})" for n, c in sorted(counts.items(), key=lambda i: (-i[1], i[0]))[:15]] return (f"{ROLE_PLURAL[target]} prác ({describe(query)}; {len(rows)} prác v CRZP):\n\n" + ("\n".join(lines) or "- CRZP ich neuvádza")), ranked(question, query, rows)[:10] @@ -565,8 +694,8 @@ def _similar(question, query, found, rows) -> tuple[str, list[dict]]: if not pairs: return (f"K práci **{work['title']}** som medzi prácami s jej kľúčovými slovami nenašiel inú so spoločnými " f"kľúčovými slovami. · {_link(work)}"), [work] - lines = [f"- **{c['title']}** ({_kind(c)}, {c.get('year') or '?'}, {school(c)}) — spoločné: " - f"{', '.join(shared)} · {_link(c)}" for c, shared in pairs] + lines = [f"- {_names(c.get('authors', []))} — {university(c)}: „{c['title']}“ ({_kind(c)}, {c.get('year') or '?'}); " + f"spoločné: {', '.join(shared)} · {_link(c)}" for c, shared in pairs] text = (f"Práce so spoločnými kľúčovými slovami CRZP ako **{work['title']}** ({_link(work)}); porovnané s prácami " f"nájdenými podľa kľúčových slov „{'“, „'.join(searched)}“:\n\n" + "\n".join(lines)) return text, [work] + [c for c, _ in pairs] @@ -663,7 +792,7 @@ def _answer_query(question: str, query: CrzpQuery, complete=None) -> dict: return _result(question, "no_answer", "Na vyhľadanie v CRZP uveď meno študenta, vedúceho alebo oponenta, " "názov práce alebo tému.", reason="missing_search_terms") try: - found, rows = find(query, links=query.intent != "count") + found, rows = find(query) if found is None: return _result(question, "no_answer", "Otázke nezodpovedá žiadne vyhľadávanie v CRZP.", reason="missing_search_terms") @@ -703,9 +832,9 @@ def _answer_query(question: str, query: CrzpQuery, complete=None) -> dict: terms = _without_suffix(query, found["texts"]) if len(terms) > 1 and query.topics: body += "\n\nHľadané výrazy v CRZP: „" + "“, „".join(terms) + "“ (vrátane príbuzných výrazov témy)." - truncated = not found["complete"] + truncated = not found["complete"] and not found.get("facet_only") # a CRZP total needs no read rows reasons = ["crzp_result_limit"] if truncated else [] - for text, total, read in found["incomplete"]: + for text, total, read in (found["incomplete"] if truncated else []): body += (f"\n\nPoznámka: hľadanie „{text}“ vrátilo v CRZP {total} záznamov; prehľadaných bolo prvých {read}.") body += "\n\nZdroj: Centrálny register záverečných prác (CRZP), metadáta záznamov." result = _result(question, "ok", body, rows=used, complete=not truncated, truncated=truncated, reasons=reasons, diff --git a/tools/crzp/answering/context.py b/tools/crzp/answering/context.py index bb6c199..31ef7c2 100644 --- a/tools/crzp/answering/context.py +++ b/tools/crzp/answering/context.py @@ -26,7 +26,7 @@ SCHEMA = "crzp-1" LIFETIME = 24 * 60 * 60 MAX_TOKEN = 16000 MAX_SIDS = 30 -CARRIED = {"intent", "people", "title", "topics", "surface", "phrases", "school_name", "years", "work_types", "school", "unit", +CARRIED = {"intent", "people", "title", "topics", "surface", "phrases", "school_name", "works", "years", "work_types", "school", "unit", "alternatives", "target_role", "from_model", "sids"} CONTINUES = re.compile(r"^(?:a|aj|a co|co tak|a ako|a kto|a kolko|a ktore|a v|a z|a pre|a len|iba|len|teraz)\b") @@ -144,6 +144,7 @@ def follow_up(question: str, new: CrzpQuery, previous: dict) -> tuple[CrzpQuery, if chosen: merged.people[0].name = merged.people[0].typed = chosen["name"] merged.school_name = chosen["school"] + merged.works = list(chosen.get("works") or []) merged.from_model = True # the exact CRZP name: no other name forms return merged, None if ordinal is not None and not own_subject: diff --git a/tools/crzp/answering/lexicon.py b/tools/crzp/answering/lexicon.py new file mode 100644 index 0000000..3f2eaba --- /dev/null +++ b/tools/crzp/answering/lexicon.py @@ -0,0 +1,60 @@ +"""CRZP's own vocabulary for questions answered without the model. + +CRZP matches exact words, so an inflected topic ("rozpoznávaním emócií") must +be turned into the form the records use. The tags (keywords) of every record +read before are counted; an inflected phrase is mapped to the most frequent +tag with the same number of words whose words match it one by one up to +inflection ("rozpoznávanie emócií", "neurónové siete"). Nothing is invented: +only phrases CRZP records really carry are used. +""" +from __future__ import annotations + +import threading +import time +from collections import Counter + +from tools.crzp.source.parse import fold + +REFRESH_SECONDS = 3600 +_lock = threading.Lock() +_state = {"built": 0.0, "phrases": {}} + + +def _stem_equal(a: str, b: str) -> bool: + n = max(3, min(len(a), len(b)) - 2) + return len(a) >= 3 and len(b) >= 3 and a[:n] == b[:n] and abs(len(a) - len(b)) <= 4 + + +def _build() -> dict: + from tools.crzp.fetch import store + counts, spelled = Counter(), {} + for row in store().cached_rows(): + for tag in row.get("keywords", []): + key = " ".join(fold(tag).split()) + if 1 <= len(key.split()) <= 4 and len(key) <= 60: + counts[key] += 1 + spelled.setdefault(key, tag.strip()) + by_words: dict = {} + for key, count in counts.items(): + by_words.setdefault((len(key.split()), key.split()[0][:3]), []).append((count, key)) + return {"by_words": by_words, "spelled": spelled} + + +def vocabulary() -> dict: + with _lock: + if time.monotonic() - _state["built"] > REFRESH_SECONDS: + _state["phrases"] = _build() + _state["built"] = time.monotonic() + return _state["phrases"] + + +def tag_for(words: list[str]) -> str | None: + """The most frequent CRZP tag matching the inflected words one by one, or None.""" + folded = [fold(w) for w in words if w] + if not folded or len(folded) > 4: + return None + table = vocabulary() + candidates = table.get("by_words", {}).get((len(folded), folded[0][:3]), []) + best = max(((count, key) for count, key in candidates + if all(_stem_equal(a, b) for a, b in zip(folded, key.split()))), default=None) + return table["spelled"][best[1]] if best else None diff --git a/tools/crzp/answering/understand.py b/tools/crzp/answering/understand.py index cac9790..dbfe7fc 100644 --- a/tools/crzp/answering/understand.py +++ b/tools/crzp/answering/understand.py @@ -105,6 +105,7 @@ class CrzpQuery(BaseModel): relaxed_author: bool = False # asked as author, found only in another role sids: list[str] = Field(default_factory=list, max_length=30) # follow-up: only these CRZP records school_name: str | None = None # follow-up: the chosen namesake's university (answer.university) + works: list[str] = Field(default_factory=list, max_length=30) # follow-up: the chosen namesake's works searched: list[str] = Field(default_factory=list) # texts actually sent to CRZP (set by answer.find) unverified_only: list[str] = Field(default_factory=list) # found only under unverified proposed terms @@ -173,6 +174,8 @@ def asks_for_works(low: str) -> bool: def _intent(low: str) -> Intent: if OUT_OF_SCOPE.search(fold(low)): return "refused" + if re.search(r"^\W*kto\b[^?]*\b(?:viedol|viedla|viedli|oponoval|oponovala|oponovali|[sš]kolil)\w*\b[^?]*\bpr[aá]c", low): + return "relation" # "Kto viedol najviac prác o X?": the supervisors, not a count if re.search(r"\bkoľk\w*|\bkolk\w*|\bpočet\b|\bpocet\b|\bhow many\b", low): return "count" if re.search(r"kľúčov|klucov|\btag|keyword", low): @@ -264,6 +267,18 @@ def _rule_people_topics(question: str, query: CrzpQuery) -> None: query.topics = list(dict.fromkeys(topics))[:4] query.surface = [" ".join(surface[:2])] if surface else [] query.phrases = topic_phrases(runs[0])[:3] if runs else [] + if runs: # the tag CRZP records really use for these words comes first + from tools.crzp.answering.lexicon import tag_for + try: + tag = tag_for([token for token, _ in runs[0]]) + except Exception: + tag = None + if tag: + query.phrases = [tag, *query.phrases] + unique = {} + for phrase in query.phrases: # CRZP ignores diacritics: "rozpoznavanie" is "rozpoznávanie" + unique.setdefault(" ".join(fold(phrase).split()), phrase) + query.phrases = list(unique.values())[:3] PLURAL_NOUN = [("iach", "ie"), ("ach", "e"), ("och", "y"), ("ami", "e"), ("mi", "y"), ("ov", "y"), ("í", "e")] @@ -351,7 +366,7 @@ def understand(question: str, *, complete=None, use_model: bool = True) -> CrzpQ query.topics = [t.strip() for t in proposed.get("topics") or [] if isinstance(t, str) and t.strip() and in_question(t, question) and not work_words_only(t) and not names_a_school(t) and not any(in_question(t, p.name) for p in query.people)][:4] - if (query.intent == "list" and proposed.get("intent") in ("count", "about", "keywords", "relation", "similar", "schools") + if (query.intent in ("list",) and proposed.get("intent") in ("count", "about", "keywords", "relation", "similar", "schools") and not (proposed["intent"] == "about" and asks_for_works(question.lower()))): query.intent = proposed["intent"] # rules found no cue (e.g. another language) title = proposed.get("title") @@ -366,6 +381,12 @@ def understand(question: str, *, complete=None, use_model: bool = True) -> CrzpQ offered = [] if query.topics and not query.people and not query.title: query.alternatives = alternatives(query.topic_key(), offered) + who = re.match(r"^\W*kto\b[^?]*?\b(viedol|viedla|viedli|oponoval|oponovala|oponovali|[sš]kolil)", question.lower()) + if who and query.intent == "relation": # "Kto oponoval práce Hládka?": the opponents of Hládek's works + query.target_role = "opponent" if who.group(1).startswith("opon") else "supervisor" + for person in query.people: + if person.role == query.target_role: + person.role = "any" if query.target_role is None and query.intent == "relation": asked = [r for r, cue in ROLE_CUES.items() if re.search(cue, question.lower()) and all(p.role != r for p in query.people)] diff --git a/tools/crzp/answering/units.py b/tools/crzp/answering/units.py index 5b2428e..e1873f8 100644 --- a/tools/crzp/answering/units.py +++ b/tools/crzp/answering/units.py @@ -13,15 +13,20 @@ Older records (about until 2015) write the department by name ("1040 Katedra elektroniky a multimediálnych telekomunikácií") instead of 104005; CRZP indexes that text too, so a workplace is searched by every form in "search" and the name is matched as written. -Other codes (e.g. 104009, 104004) are shown as numbers, never guessed. +Other TUKE department codes are named only when workplaces.json holds a name +learned from CRZP records with enough votes (see workplaces.py); such a +workplace can be asked by its acronym ("na KPI") or name and is searched by +its code and its old written name. Codes without evidence stay numbers +("pracovisko 104009"), never guessed. """ from __future__ import annotations import re +from tools.crzp.answering import workplaces from tools.crzp.source.parse import fold -UNITS = { +VERIFIED = { "kemt": {"school": "tuke", "code": "104005", "label": "KEMT FEI", "text": "katedra elektroniky a multimedialnych telekomunikacii", "search": ["104005", "Katedra elektroniky a multimediálnych telekomunikácií"]}, @@ -34,10 +39,28 @@ CUES = { } +def _learned_units() -> dict: + """Departments named in workplaces.json, keyed by their acronym ("kpi").""" + units = {} + for short, (code, entry) in workplaces.by_acronym(workplaces.TUKE).items(): + if short in VERIFIED or code in {u["code"] for u in VERIFIED.values()}: + continue + units[short] = {"school": "tuke", "code": code, "label": f"{entry['short']} FEI" if code.startswith("1040") + else entry["short"], "text": fold(entry["name"]), "search": [code, entry["name"]], + "learned": True} + words = [w for w in fold(entry["name"]).split() if w not in workplaces.SMALL_WORDS] + CUES[short] = r"\b" + re.escape(short) + r"\b|" + r"\w*\s+(?:\w+\s+)?".join(re.escape(w[:5]) for w in words) + return units + + +UNITS = {**VERIFIED, **_learned_units()} +ORDER = ["kemt", *[k for k in UNITS if UNITS[k].get("learned")], "fei"] # the most specific unit first + + def unit_in_question(question: str) -> str | None: - """The most specific verified unit the question names (KEMT before FEI).""" - low = question.lower() - return next((key for key in ("kemt", "fei") if re.search(CUES[key], low)), None) + """The most specific known unit the question names (a department before FEI).""" + low = fold(question) + return next((key for key in ORDER if re.search(fold(CUES[key]) if key in ("kemt", "fei") else CUES[key], low)), None) def _tuke(row: dict) -> bool: @@ -64,6 +87,7 @@ def unit_label(row: dict) -> str: part = part.strip(" ,") if not part: continue - known = next((u["label"].split()[0] for u in UNITS.values() if u["code"] == part), None) - names.append(known or (f"pracovisko {part}" if part.isdigit() else part)) + known = next((u["label"].split()[0] for u in VERIFIED.values() if u["code"] == part), None) + entry = workplaces.name_of(workplaces.TUKE, part) if part.isdigit() and not known else None + names.append(known or (f"{entry['name']} ({part})" if entry else f"pracovisko {part}" if part.isdigit() else part)) return ", ".join(names) diff --git a/tools/crzp/answering/wiki_bridge.py b/tools/crzp/answering/wiki_bridge.py new file mode 100644 index 0000000..559a885 --- /dev/null +++ b/tools/crzp/answering/wiki_bridge.py @@ -0,0 +1,88 @@ +"""ZP Wiki -> CRZP: a thesis fact the wiki profile lacks, taken from CRZP. + +A student's ZP Wiki profile may be incomplete (no opponent, no keywords, no +abstract, no final title). When /rag or /graphrag answers "no_answer" for a +person the wiki knows, the same question is asked in CRZP for exactly that +person (the canonical wiki name) at Technická univerzita v Košiciach, the school +of ZP Wiki. The answer keeps the wiki's own message first and marks the CRZP +part with its source, so it is never presented as a wiki fact. + +Not used for what CRZP does not hold (meetings, progress, tasks, assignments, +results beyond the abstract, repositories) and never for an unknown or +ambiguous person, a refused question or another failure. CRZP_WIKI_FALLBACK=0 +turns it off. +""" +from __future__ import annotations + +import os +import re + +from tools.answers.context_policy import normalized +from tools.answers.delivery_contract import fingerprint + +# What only the wiki records: CRZP is not asked. +WIKI_ONLY = re.compile(r"\b(?:vysled\w*|stretnut\w*|pokrok\w*|progres\w*|uloh\w*|zadan\w*|plan\w*|termin\w*|" + r"tyzd\w*|repozitar\w*|git\w*|kod\w*|stav\w*|poznam\w*|konzult\w*|zaciat\w*|studi\w*|" + r"obhaj\w*|obhaj\w*|ukonc\w*|dokonc\w*|skoncil\w*|uspes\w*|statnic\w*|hodnot\w*|znamk\w*)\b") +# A CRZP record means a submitted work, not a defended or graded one: completion stays a wiki question. +INTENTS = ("list", "about", "keywords", "relation", "count") + + +def enabled() -> bool: + return os.getenv("CRZP_WIKI_FALLBACK", "1") != "0" + + +def wiki_person(question: str, db_file=None) -> str | None: + """The one ZP Wiki person the question names (canonical name), else None.""" + match = wiki_match(question, db_file) + return match["name"] if match else None + + +def wiki_match(question: str, db_file=None) -> dict | None: + """{"name", "student"}: the one ZP Wiki person the question names; student = has a profile page.""" + from tools.answers.question_plan import planning_catalog + from tools.rag.common import DB_FILE + from tools.retrieval.person_names import resolve_person_names + try: + catalog = planning_catalog(db_file or DB_FILE) + except (OSError, ValueError, RuntimeError): + return None + resolution = resolve_person_names(question, catalog or []) + matches = resolution.get("matches") or [] + if resolution.get("status") != "ok" or len({m["name"] for m in matches}) != 1: + return None + source = matches[0].get("source_document") or "" + return {"name": matches[0]["name"], "student": source.startswith(("pages/students/", "pages/interns/"))} + + +def supplement(question: str, wiki_result: dict, engine: str, *, db_file=None, complete=None) -> dict | None: + """The wiki result extended by the CRZP answer for the same person, or None.""" + if not enabled() or wiki_result.get("status") != "no_answer" or WIKI_ONLY.search(normalized(question)): + return None + person = wiki_person(question, db_file) + if not person: + return None + from tools.crzp.answering import answer as crzp + from tools.crzp.answering.understand import Person, understand + query = understand(question, complete=complete) + if query.intent not in INTENTS: + return None + role = query.people[0].role if query.people else "any" + query.people = [Person(name=person, role=role, typed=person)] + query.from_model = True # the canonical wiki name: no other name forms + query.school, query.school_name = "tuke", None + query.title = query.title if query.title and person not in query.title else None + result = crzp._answer_query(question, query, complete) + if result.get("status") != "ok": + return None + wiki_text = (wiki_result.get("answer") or "").strip() + text = ((wiki_text + "\n\n" if wiki_text else "") + "**Doplnené z CRZP** (Centrálny register záverečných prác, " + f"Technická univerzita v Košiciach; ZP Wiki tento údaj neuvádza):\n\n" + result["answer"]) + truncated = bool(result.get("truncated")) + return {**{k: v for k, v in wiki_result.items() if k not in ("answer", "reason", "delivery_check")}, + "status": "ok", "reason": None, "accepted_for_delivery": True, "answer": text, + "source_urls": list(dict.fromkeys([*(wiki_result.get("source_urls") or []), *result["source_urls"]])), + "claims": [], "complete": not truncated, "truncated": truncated, + "truncation_reasons": result.get("truncation_reasons", []), "supplement": "crzp", + "delivery_check": {"layer": "delivery", "code": "partial" if truncated else "complete", + "request_fingerprint": fingerprint(question, engine)}} diff --git a/tools/crzp/answering/workplaces.json b/tools/crzp/answering/workplaces.json new file mode 100644 index 0000000..b3616d1 --- /dev/null +++ b/tools/crzp/answering/workplaces.json @@ -0,0 +1,144 @@ +{ + "Technická univerzita v Košiciach": { + "101201": { + "lead": 6, + "name": "Ústav podnikania a manažmentu", + "short": "ÚPM", + "voters": 6, + "votes": 6 + }, + "101401": { + "lead": 4, + "name": "Ústav montánnych vied a ochrany životného prostredia", + "short": "ÚMVOŽP", + "voters": 6, + "votes": 5 + }, + "101601": { + "lead": 5, + "name": "Ústav logistiky priemyslu a dopravy", + "short": "ÚLPD", + "voters": 5, + "votes": 5 + }, + "103102": { + "lead": 4, + "name": "Katedra aplikovanej mechaniky a mechatroniky", + "short": "KAMM", + "voters": 4, + "votes": 4 + }, + "103201": { + "lead": 6, + "name": "Katedra technológií a materiálov", + "short": "KTM", + "voters": 6, + "votes": 6 + }, + "103203": { + "lead": 6, + "name": "Katedra manažmentu a ekonomiky", + "short": "KME", + "voters": 6, + "votes": 6 + }, + "103402": { + "lead": 5, + "name": "Katedra environmentalistiky a riadenia procesov", + "short": "KERP", + "voters": 5, + "votes": 5 + }, + "104004": { + "lead": 6, + "name": "Katedra kybernetiky a umelej inteligencie", + "short": "KKUI", + "voters": 6, + "votes": 6 + }, + "104007": { + "lead": 5, + "name": "Katedra elektroenergetiky", + "short": "KE", + "voters": 5, + "votes": 5 + }, + "104009": { + "lead": 6, + "name": "Katedra počítačov a informatiky", + "short": "KPI", + "voters": 6, + "votes": 6 + }, + "105201": { + "lead": 5, + "name": "Ústav budov a prostredia", + "short": "ÚBP", + "voters": 5, + "votes": 5 + }, + "106003": { + "lead": 4, + "name": "Katedra výrobných technológií", + "short": "KVT", + "voters": 4, + "votes": 4 + }, + "106005": { + "lead": 2, + "name": "Katedra manažmentu výroby", + "short": "KMV", + "voters": 5, + "votes": 3 + }, + "107002": { + "lead": 3, + "name": "Katedra bankovníctva a investovania", + "short": "KBI", + "voters": 5, + "votes": 4 + }, + "107003": { + "lead": 2, + "name": "Katedra ekonomických teórií", + "short": "KET", + "voters": 4, + "votes": 3 + }, + "108003": { + "lead": 4, + "name": "Katedra architektúry", + "short": "KA", + "voters": 4, + "votes": 4 + }, + "109003": { + "lead": 2, + "name": "Katedra avioniky", + "short": "KA", + "voters": 4, + "votes": 3 + }, + "109005": { + "lead": 2, + "name": "Katedra letovej prípravy", + "short": "KLP", + "voters": 4, + "votes": 3 + }, + "109006": { + "lead": 3, + "name": "Katedra leteckého inžinierstva", + "short": "KLI", + "voters": 5, + "votes": 4 + }, + "109007": { + "lead": 5, + "name": "Katedra manažmentu leteckej prevádzky", + "short": "KMLP", + "voters": 5, + "votes": 5 + } + } +} diff --git a/tools/crzp/answering/workplaces.py b/tools/crzp/answering/workplaces.py new file mode 100644 index 0000000..7af9be4 --- /dev/null +++ b/tools/crzp/answering/workplaces.py @@ -0,0 +1,180 @@ +"""Names of CRZP workplace codes, learned from CRZP records themselves. + +CRZP writes newer works as "Technická univerzita v Košiciach 1040 104009" and +does not publish what 104009 means. Older records (about until 2015) write the +department by name ("1040 Katedra počítačov a informatiky"). A supervisor stays +at a department, so the name is learned per code: + +1. search the code; the full supervisor facet of that result gives the people + who supervise works with this code; +2. for the most frequent supervisors read their older works (OLD_YEARS) at the + same university and collect the department names written there; +3. each supervisor votes once, for the department named most often in his or + her older works (spellings cut by CRZP, "… umelej inteligenci", count as the + full name); +4. accept a name only when at least MIN_VOTES supervisors agree, they are at + least MIN_SHARE of the voters and the name leads the next one by MARGIN votes; +5. a name that wins for several codes is kept only for the code with the + clearest vote (the others stay numbers); placeholder codes (999999) are skipped. + +The result is stored in workplaces.json (committed, reviewable) with its votes, +so a code is named only with evidence; other codes stay "pracovisko 104009". + + python -m tools.crzp.answering.workplaces learn # TUKE codes seen in the cache + python -m tools.crzp.answering.workplaces learn 104009 104004 +""" +from __future__ import annotations + +import argparse +import json +import re +from collections import Counter +from functools import lru_cache +from pathlib import Path + +from tools.crzp.source.parse import display_name, fold + +FILE = Path(__file__).with_name("workplaces.json") +TUKE = "Technická univerzita v Košiciach" +OLD_YEARS = (2005, 2015) +SUPERVISORS = 6 +MIN_VOTES = 3 +MIN_SHARE = 0.6 +MARGIN = 2 +PLACEHOLDERS = {"999999", "000000"} +SMALL_WORDS = {"a", "v", "vo", "pre", "na", "so", "s", "z", "zo", "k", "ku", "o", "a."} + + +@lru_cache(maxsize=1) +def learned() -> dict: + """{university: {code: {"name", "short", "votes", "voters"}}} from workplaces.json.""" + try: + return json.loads(FILE.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + +def name_of(university: str, code: str) -> dict | None: + return learned().get(university, {}).get(code) + + +def acronym(name: str) -> str: + """"Katedra počítačov a informatiky" -> "KPI".""" + return "".join(w[0].upper() for w in re.findall(r"[^\W\d_]+", name) if w.lower() not in SMALL_WORDS) + + +def by_acronym(university: str = TUKE) -> dict[str, tuple[str, dict]]: + """Lower-case acronym -> (code, entry) for acronyms that name exactly one learned workplace.""" + seen: dict[str, list] = {} + for code, entry in learned().get(university, {}).items(): + seen.setdefault(entry["short"].lower(), []).append((code, entry)) + return {k: v[0] for k, v in seen.items() if len(v) == 1 and len(k) >= 3} + + +CUT_WORDS = {"inteligenci": "inteligencie"} # CRZP cut old names at 40 characters + + +def _department(unit: str) -> str | None: + """The written department of an old record: "1040 Katedra počítačov a informatiky".""" + text = re.sub(r"^\s*[\d\s]+", "", unit or "").strip(" ,") + text = re.sub(r"^(Katedra|Ústav)(?=[a-záäčďéíĺľňóôŕšťúýž])", r"\1 ", text) # "Katedraaplikovanej" + words = text.split() + if words and words[-1] in CUT_WORDS: + text = " ".join([*words[:-1], CUT_WORDS[words[-1]]]) + return text if re.search(r"(?i)\b(katedr|ústav|ustav|centrum|pracovisk)\w*", text) else None + + +def codes_in_cache(university: str = TUKE) -> list[str]: + from tools.crzp.answering.answer import university as university_of + from tools.crzp.fetch import store + counts = Counter(code for row in store().cached_rows() if university_of(row) == university + for code in re.findall(r"\b\d{6}\b", row.get("school_unit") or "")) + return [code for code, _ in counts.most_common()] + + +def learn_code(code: str, university: str = TUKE, *, search=None) -> dict: + """Votes for the department name of one code (see the module docstring).""" + from tools.crzp import fetch + from tools.crzp.answering.answer import university as university_of + search = search or fetch.search + result = search(code, links=False, depth=20) + supervisors = [display_name(f["label"]) for f in result["facets"].get("Školiteľ", [])][:SUPERVISORS] + votes, voters, spelled = Counter(), 0, {} + for name in supervisors: + old = search(name, years=OLD_YEARS, links=False, depth=200) + surname = fold(name.split()[-1]) + own = Counter(_department(r.get("school_unit")) for r in old["rows"] if university_of(r) == university + and any(surname in fold(s) for s in r.get("supervisors", []))) + own.pop(None, None) + merged = Counter() + for department, n in own.items(): + key = _canonical(department, spelled) + merged[key] += n + ranked = merged.most_common(2) + if ranked and (len(ranked) == 1 or ranked[0][1] > ranked[1][1]): # one clear department of this person + voters += 1 + votes[ranked[0][0]] += 1 + if not votes: + return {"code": code, "accepted": False, "voters": voters, "votes": {}} + final = Counter() # a key may have grown to a longer spelling after it got votes + for key, n in votes.items(): + final[next((k for k in spelled if k.startswith(key)), key)] += n + votes = final + ranked = votes.most_common(2) + best, count = ranked[0] + second = ranked[1][1] if len(ranked) > 1 else 0 + accepted = count >= MIN_VOTES and count / voters >= MIN_SHARE and count - second >= MARGIN + best = spelled[best] + return {"code": code, "accepted": accepted, "name": best, "short": acronym(best), + "votes": {spelled[k]: v for k, v in votes.items()}, "lead": count - second, + "voters": voters, "supervisors": supervisors} + + +def _canonical(department: str, spelled: dict) -> str: + """Folded key shared by a name and its cut spellings; `spelled` keeps the longest form.""" + folded = " ".join(fold(department).replace("katedraa", "katedra a").split()) + for key in list(spelled): + if key.startswith(folded) or folded.startswith(key): + longer = folded if len(folded) > len(key) else key + text = department if len(folded) > len(key) else spelled[key] + if longer != key: + spelled[longer] = text + spelled.pop(key) + return longer + spelled[folded] = department + return folded + + +def learn(codes: list[str], university: str = TUKE) -> dict: + report = [learn_code(code, university) for code in codes if code not in PLACEHOLDERS] + winners: dict = {} + for outcome in report: + if outcome["accepted"] and outcome["code"] != "104005": # KEMT is verified by hand (units.py) + key = fold(outcome["name"]) + if key not in winners or (outcome["lead"], outcome["voters"]) > (winners[key]["lead"], winners[key]["voters"]): + winners[key] = outcome + table = {o["code"]: {"name": o["name"], "short": o["short"], "votes": o["votes"][o["name"]], + "voters": o["voters"], "lead": o["lead"]} for o in winners.values()} + data = {**learned(), university: dict(sorted(table.items()))} + FILE.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + learned.cache_clear() + return {"learned": len(table), "report": report} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("learn") + run.add_argument("codes", nargs="*") + run.add_argument("--limit", type=int, default=25) + args = parser.parse_args() + codes = args.codes or ["104005", *[c for c in codes_in_cache() if c != "104005"][: args.limit]] + result = learn(codes) + for line in result["report"]: + print(json.dumps({k: line.get(k) for k in ("code", "accepted", "name", "voters", "lead", "votes")}, ensure_ascii=False)) + print(json.dumps({"learned": result["learned"]})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/crzp/fetch.py b/tools/crzp/fetch.py index 38064e8..fabfedb 100644 --- a/tools/crzp/fetch.py +++ b/tools/crzp/fetch.py @@ -19,11 +19,13 @@ import threading import time from tools.crzp.source.client import CrzpSession, CrzpUnavailable, PERMALINK -from tools.crzp.source.parse import fold, parse_detail, parse_results, parse_rows, parse_rss +from tools.crzp.source.parse import facet_actions, fold, parse_detail, parse_results, parse_rows, parse_rss from tools.crzp.storage.cache import CrzpStore LINKED_ROWS = 200 # the RSS export of CRZP is capped at 200 items -DEEP_ROWS = int(os.getenv("CRZP_DEEP_ROWS", "1000")) # counts and summaries: up to 10 pages of 100 +DEEP_ROWS = int(os.getenv("CRZP_DEEP_ROWS", "3000")) # summaries: read in year ranges, several at once +PARALLEL = int(os.getenv("CRZP_PARALLEL", "3")) # year ranges read at the same time (shared throttle) +EXPANDED = ("Rok odovzdania", "Školiteľ") # facets read in full when a result exceeds one page MAX_ROWS = LINKED_ROWS PREPARED_SECONDS = 120 _prepared: list = [] # [(created, session with the home page loaded)] @@ -139,12 +141,23 @@ def search(text: str, *, years: tuple[int, int] | None = None, work_types: list[ return {**result, "cached": False} +def _expanded(session, first: dict, actions: dict) -> dict: + """All values of the year and supervisor facets (two requests), exact over the whole result.""" + full = {} + for title in EXPANDED: + if title in actions and len(first["facets"].get(title, [])) >= 10: # CRZP shows the top 10 only + full[title] = session.show_all(actions[title]) + elif title in first["facets"]: + full[title] = first["facets"][title] + return full + + def _search_live(text, years, work_types, links, session_factory, depth=LINKED_ROWS) -> dict: - depth = max(depth, LINKED_ROWS) with session_factory() as session: first = parse_results(session.search(text, years=years, work_types=work_types)) + actions = facet_actions(getattr(session, "content", "") or "") # ids from the first result page rows = first["rows"] - if first["total"] > len(rows): # up to 20 results: one page, no paging requests + if first["total"] > len(rows) and depth > len(rows): # up to 20 results: one page, no paging requests rows = parse_rows(session.set_page_size()) while len(rows) < min(first["total"], depth): page = session.next_page() @@ -157,6 +170,8 @@ def _search_live(text, years, work_types, links, session_factory, depth=LINKED_R rows = rows[:depth] if rows and links: _attach_links(rows[:LINKED_ROWS], parse_rss(session.rss_links())) + if first["total"] > len(first["rows"]): # last: after an expansion CRZP pages no more rows + first["facets"] = {**first["facets"], **_expanded(session, first, actions)} return {"query": text, "years": list(years) if years else None, "work_types": work_types or None, "total": first["total"], "facets": first["facets"], "rows": rows, "complete": len(rows) >= first["total"], "links": links} @@ -177,3 +192,64 @@ def detail(sid: str, *, session_factory=CrzpSession, cache: CrzpStore | None = N record["url"] = PERMALINK.format(sid=record["sid"]) cache.put_detail(sid, record) return record + + +_chunk_pool = {"pool": None} + + +def _pool(): + """One shared pool for year-range reads (no new threads per search).""" + from concurrent.futures import ThreadPoolExecutor + with _store_lock: + if _chunk_pool["pool"] is None: + _chunk_pool["pool"] = ThreadPoolExecutor(max_workers=max(1, PARALLEL), thread_name_prefix="crzp-range") + return _chunk_pool["pool"] + + +def year_counts(result: dict) -> dict[int, int]: + return {int(f["label"]): f["count"] for f in result["facets"].get("Rok odovzdania", []) + if str(f["label"]).isdigit()} + + +def plan(counts: dict[int, int], years, cap: int, limit: int) -> list[tuple[int, int]]: + """Year ranges, newest first, each with at most `cap` records (one year may exceed it), + until `limit` records are planned.""" + low, high = years or (min(counts), max(counts)) + chunks, current, size, planned = [], [], 0, 0 + for year, count in sorted(((y, n) for y, n in counts.items() if low <= y <= high), reverse=True): + if planned >= limit: + break + if current and size + count > cap: + chunks.append((min(current), max(current))) + current, size = [], 0 + current.append(year) + size += count + planned += count + if current: + chunks.append((min(current), max(current))) + return chunks + + +def search_deep(text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None, + links: bool = False, limit: int = DEEP_ROWS, cache: CrzpStore | None = None) -> dict: + """Up to `limit` rows of a result larger than one read, read in year ranges + (the year facet tells how many records each year has), PARALLEL ranges at once. + With links, every range has at most LINKED_ROWS records, so its RSS export + gives a permalink to every row; without links the ranges are larger.""" + head = search(text, years=years, work_types=work_types, links=links, + depth=LINKED_ROWS if links else 20, cache=cache) + if head["complete"]: + return head + counts = year_counts(head) + if not counts: + return search(text, years=years, work_types=work_types, links=links, depth=limit, cache=cache) + wanted = min(head["total"], limit) + cap = LINKED_ROWS if links else max(LINKED_ROWS, -(-wanted // PARALLEL)) + chunks = plan(counts, years, cap, limit) + parts = list(_pool().map(lambda c: search(text, years=c, work_types=work_types, links=links, + depth=LINKED_ROWS if links else limit, cache=cache), chunks)) + rows = [r for part in parts for r in part["rows"]][:limit] + read_total = sum(part["total"] for part in parts) + return {**head, "rows": rows, "complete": len(rows) >= head["total"], "chunks": chunks, + "links": links, "cached": all(part.get("cached") for part in parts), + "chunks_complete": all(part["complete"] for part in parts) and read_total >= head["total"]} diff --git a/tools/crzp/source/client.py b/tools/crzp/source/client.py index 2c5baa1..217dc10 100644 --- a/tools/crzp/source/client.py +++ b/tools/crzp/source/client.py @@ -6,8 +6,10 @@ replays exactly what the browser sends (no JavaScript engine is needed): search with year and work-type filters, change the page size, go to the next page and export the permalinks (RSS); a record detail is read through its public permalink. Only metadata pages are read; full texts and reviews never. -All requests of the process share one throttle (CRZP answers bursts with -HTTP 429), so concurrency cannot raise the request rate. +All requests of the process share one throttle: at most one request start per +CRZP_MIN_INTERVAL_SECONDS (0.5 s, i.e. 2 requests/s; one CRZP page takes about +1.1 s, so only reads of several year ranges at once overlap). An HTTP 429 slows +every request down to CRZP_PENALTY_SECONDS (2 s) for CRZP_PENALTY_DURATION (300 s). """ from __future__ import annotations @@ -33,14 +35,24 @@ class CrzpUnavailable(RuntimeError): class _Throttle: - """Minimum interval between any two requests of this process.""" + """Minimum interval between any two request starts of this process; slower after a 429.""" def __init__(self): self._lock = threading.Lock() self._last = 0.0 + self._penalty_until = 0.0 + + def penalize(self): + self._penalty_until = time.monotonic() + float(os.getenv("CRZP_PENALTY_DURATION", "300")) + + def interval(self) -> float: + interval = float(os.getenv("CRZP_MIN_INTERVAL_SECONDS", "0.5")) + if time.monotonic() < self._penalty_until: + interval = max(interval, float(os.getenv("CRZP_PENALTY_SECONDS", "2.0"))) + return interval def wait(self): - interval = float(os.getenv("CRZP_MIN_INTERVAL_SECONDS", "1.0")) + interval = self.interval() with self._lock: delay = self._last + interval - time.monotonic() if delay > 0: @@ -87,7 +99,7 @@ class CrzpSession: """One OPAC session (cookie + current page). Not shared between threads.""" def __init__(self, timeout: float | None = None): - timeout = timeout or float(os.getenv("CRZP_TIMEOUT_SECONDS", "20")) + timeout = timeout or float(os.getenv("CRZP_TIMEOUT_SECONDS", "30")) self.http = httpx.Client(base_url=BASE_URL, headers={"User-Agent": USER_AGENT}, follow_redirects=True, timeout=timeout) self.page = "" # last full page: form target and hidden fields @@ -115,11 +127,15 @@ class CrzpSession: try: response = self.http.request(method, url, **kwargs) except httpx.HTTPError as error: - raise CrzpUnavailable("network_error") from error + if attempt == 2: + raise CrzpUnavailable("network_error") from error + time.sleep(1.0 + attempt) # one slow or dropped answer is repeated, not reported + continue if response.status_code != 429: if response.status_code >= 400: raise CrzpUnavailable(f"http_{response.status_code}") return response + THROTTLE.penalize() retry = response.headers.get("Retry-After", "") time.sleep(min(float(retry) if retry.isdigit() else 3.0 * (attempt + 1), 10.0)) raise CrzpUnavailable("rate_limited") @@ -199,6 +215,20 @@ class CrzpSession: match = re.search(r"ae\('(\w+)','(navigator:navBiblio_\d+)'\);\" title=\"\+1\"", self.content) return self.act(match.group(1), match.group(2)) if match else None + def show_all(self, element: str) -> list[dict]: + """Every value of a facet ("Rok odovzdania", "Školiteľ"): [{label, count}] (one request).""" + script = self.act(element, "showMoreResultsNewAjax") + if "[" not in script: # right after another expansion CRZP first only renews the action GUID + script = self.act(element, "showMoreResultsNewAjax") + start = script.find("[") + try: + items = json.loads(script[start:script.rfind("]") + 1]) if start >= 0 else None + except ValueError: + items = None + if not isinstance(items, list) or not all(isinstance(i, dict) and "label" in i and "count" in i for i in items): + raise CrzpUnavailable("page_structure_changed") + return [{"label": str(i["label"]), "count": int(i["count"])} for i in items] + def rss_links(self) -> str: """RSS export of the current search: titles and permalinks (CRZP caps it at 200).""" element = action_element(self.content, "makeRSSChannelAjax") or action_element(self.page, "makeRSSChannelAjax") diff --git a/tools/crzp/source/parse.py b/tools/crzp/source/parse.py index 8e7deee..fd17355 100644 --- a/tools/crzp/source/parse.py +++ b/tools/crzp/source/parse.py @@ -87,6 +87,21 @@ def facets(page: str) -> dict[str, list[dict]]: return result +def facet_actions(page: str) -> dict[str, str]: + """Facet title -> id of the element that shows all of its values.""" + result = {} + for raw in re.findall(r'', page, re.S): + try: + data = json.loads(raw.replace('\\\\"', '\\"')) + except ValueError: + continue + show = data.get("onShowAll") if isinstance(data, dict) else None + if (isinstance(data.get("strings") if isinstance(data, dict) else None, dict) and isinstance(show, list) + and len(show) == 2 and isinstance(show[1], list) and show[1][1:2] == ["showMoreResultsNewAjax"]): + result[data["strings"].get("title", "")] = show[1][0] + return result + + def _lines(chunk: str, label: str) -> list[str]: return [text(v) for v in re.findall(re.escape(label) + r":\s*([^<]*)<", chunk) if text(v)] diff --git a/tools/crzp/storage/cache.py b/tools/crzp/storage/cache.py index a927483..a4f6e00 100644 --- a/tools/crzp/storage/cache.py +++ b/tools/crzp/storage/cache.py @@ -75,6 +75,20 @@ class CrzpStore: def put_detail(self, sid: str, payload: dict) -> None: self._put("details", sid, payload) + def cached_rows(self, ttl: float | None = None): + """Every row of every cached search (records read before; used for the workplace and + vocabulary tables). Rows of an older schema are skipped.""" + if not self.path.exists(): + return + with self._connect() as connection: + items = connection.execute("SELECT fetched_at, payload FROM searches").fetchall() + for fetched_at, payload in items: + if ttl is not None and time.time() - fetched_at > ttl: + continue + value = json.loads(payload) + if value.get("schema") == SCHEMA: + yield from value.get("rows", []) + def stats(self) -> dict: if not self.path.exists(): return {"searches": 0, "details": 0} diff --git a/tools/crzp/storage/graph.py b/tools/crzp/storage/graph.py index a40b716..bb4bdaa 100644 --- a/tools/crzp/storage/graph.py +++ b/tools/crzp/storage/graph.py @@ -45,6 +45,77 @@ RETURN o.sid AS sid, collect(k.label) AS shared ORDER BY size(shared) DESC, sid """ +# Links to the ZP Wiki graph, written by code only (never by the model): +# a CRZP person with a work at TUKE whose name equals a ZP Wiki person (probably the same person; CRZP +# has no identifiers), and a CRZP keyword that is a ZP Wiki topic or keyword (same words or reviewed synonyms). +LINK_PEOPLE = """ +UNWIND $pairs AS pair +MATCH (c:CrzpPerson {name: pair.crzp}) +MATCH (p:Person {name: pair.wiki}) +MERGE (c)-[r:PROBABLY_SAME]->(p) SET r.basis = 'rovnaké meno, práca na TUKE' +""" +LINK_TOPICS = """ +UNWIND $pairs AS pair +MATCH (k:CrzpKeyword {key: pair.key}) +MATCH (t) WHERE (t:Topic OR t:Keyword) AND t.name = pair.wiki +MERGE (k)-[:SAME_TOPIC]->(t) +""" +WIKI_NAMES = "MATCH (p:Person) RETURN p.name AS name" +WIKI_TOPICS = "MATCH (t) WHERE t:Topic OR t:Keyword RETURN DISTINCT t.name AS name" +PERSON_WORKS = """ +MATCH (p:Person {name: $name})<-[:PROBABLY_SAME]-(c:CrzpPerson)<-[r:AUTHOR|SUPERVISOR|OPPONENT]-(t:CrzpThesis) +RETURN t.sid AS sid, t.title AS title, t.year AS year, t.work_type AS work_type, type(r) AS role ORDER BY t.year DESC +""" +_wiki = {"at": 0.0, "names": [], "topics": []} + + +def link_pairs(rows: list[dict], wiki_names: list[str], wiki_topics: list[str]) -> tuple[list[dict], list[dict]]: + """(person pairs, keyword pairs) for the rows; pure, so it is tested without Neo4j.""" + from tools.crzp.source.parse import display_name + from tools.retrieval.synonyms import base_name + names = {fold(n): n for n in wiki_names if n} + topics = {} + for topic in wiki_topics: + if topic: + topics.setdefault(fold(topic), topic) + topics.setdefault(fold(base_name(topic)), topic) + people, keywords = {}, {} + for row in rows: + if "technicka univerzita v kosiciach" not in fold(row.get("school") or ""): + continue + for field in ("authors", "supervisors", "opponents"): + for name in row.get(field, []): + wiki = names.get(fold(display_name(name))) + if wiki: + people[name] = wiki + for row in rows: + for keyword in row.get("keywords", []): + wiki = topics.get(fold(keyword)) or topics.get(fold(base_name(keyword))) + if wiki: + keywords[fold(keyword)] = wiki + return ([{"crzp": c, "wiki": w} for c, w in people.items()], [{"key": k, "wiki": w} for k, w in keywords.items()]) + + +def link(session, rows: list[dict]) -> None: + import time + if time.time() - _wiki["at"] > 600: + _wiki.update(at=time.time(), names=session.run(WIKI_NAMES).value(), topics=session.run(WIKI_TOPICS).value()) + people, keywords = link_pairs(rows, _wiki["names"], _wiki["topics"]) + if people: + session.execute_write(lambda tx: tx.run(LINK_PEOPLE, pairs=people).consume()) + if keywords: + session.execute_write(lambda tx: tx.run(LINK_TOPICS, pairs=keywords).consume()) + + +def person_works(name: str) -> list[dict]: + """CRZP works linked to a ZP Wiki person in the graph ([] without Neo4j).""" + try: + with _session() as session: + return [dict(r) for r in session.run(PERSON_WORKS, name=name)] + except Exception: + return [] + + def _session(): global _driver from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings @@ -66,6 +137,10 @@ def project(rows: list[dict]) -> bool: try: with _session() as session: session.execute_write(lambda tx: tx.run(PROJECT, rows=data).consume()) + try: + link(session, rows) + except Exception: + logger.warning("Linking CRZP records to the ZP Wiki graph failed.") return True except Exception: logger.warning("CRZP projection to Neo4j failed; Python aggregation is used.") diff --git a/tools/crzp/tests/test_crzp.py b/tools/crzp/tests/test_crzp.py index 3486ddf..8f94aba 100644 --- a/tools/crzp/tests/test_crzp.py +++ b/tools/crzp/tests/test_crzp.py @@ -195,12 +195,16 @@ def test_about_shows_a_summary_only_after_the_check(crzp): "Autor meral teplotu senzormi ESP32 a údaje posielal cez MQTT.", '{"supported": true}']) model = lambda messages, **kw: {"choices": [{"message": {"content": next(replies)}}]} result = A._answer("O čom je práca Jána Kováča?", complete=model) - assert "Zhrnutie podľa abstraktu: Autor meral teplotu" in result["answer"] and "54 strán" in result["answer"] + assert "\n\nAutor meral teplotu senzormi ESP32" in result["answer"] and "54 strán" not in result["answer"] + assert result["answer"].startswith("**Meranie teploty cez MQTT** — bakalárska práca, 2025, Technická univerzita v Košiciach; autor Ján Kováč") A._cache.clear() replies = iter(['{"people":[{"name":"Ján Kováč","role":"author"}],"topics":[],"title":null,"target_role":null}', - "Autor vyvinul blockchain.", '{"supported": false}']) + "Autor vyvinul blockchain.", '{"supported": false}', "Autor vyvinul kvantový počítač.", + '{"supported": false}']) rejected = A._answer("O čom je práca Jána Kováča?", complete=model) - assert "Abstrakt: Práca opisuje meranie teploty" in rejected["answer"] and "blockchain" not in rejected["answer"] + # Never the copied abstract: a sentence from the CRZP keywords instead of an unsupported summary. + assert "Bakalárska práca „Meranie teploty cez MQTT“ sa zaoberá témami: iot, mqtt." in rejected["answer"] + assert "blockchain" not in rejected["answer"] and "Práca opisuje meranie teploty" not in rejected["answer"] def test_relation_aggregates_opponents_of_supervised_works(crzp): @@ -251,13 +255,13 @@ def test_client_retries_rate_limit_then_reports_it(monkeypatch): lambda request: httpx.Response(429, text="slow down"))) with pytest.raises(C.CrzpUnavailable, match="rate_limited"): session.get("/") + assert C.THROTTLE.interval() >= 2.0 # a 429 slows every next request down + C.THROTTLE._penalty_until = 0.0 -def test_crzp_is_an_answer_engine_of_the_mcp_client(): +def test_crzp_is_no_separate_tool_of_the_mcp_client(): import packages.claude.client as client - assert "crzp" in client.ENGINES - assert client.query_backend("/rag Práca?", "crzp")["reason"] == "engine_mismatch" - assert client.query_backend("Práca?", "crzp", mode="retrieve")["reason"] == "invalid_query" + assert client.ENGINES == ("rag", "graphrag") # CRZP answers inside /rag and /graphrag # ---------------------------------------------------------------- real page structure (anonymized fixtures) @@ -353,7 +357,7 @@ def test_verified_workplace_filter_and_label(crzp): assert "KEMT práca" in result["answer"] and "Iná katedra" not in result["answer"] assert "Technická univerzita v Košiciach, FEI, KEMT" in result["answer"] assert units.unit_label({"school": "Technická univerzita v Košiciach", "school_unit": "1040 104009"}) == \ - "FEI, pracovisko 104009" + "FEI, Katedra počítačov a informatiky (104009)" assert units.unit_in_question("práce z FEI") == "fei" and units.unit_in_question("práce z TUKE") is None @@ -389,9 +393,9 @@ def test_structure_check_reports_degraded_health(monkeypatch, tmp_path): def test_department_written_by_name_is_kemt_and_not_a_namesake(crzp): assert split_school("Technická univerzita v Košiciach 1040 Katedra elektroniky a multimediálnych telekomunikácií") == \ ("Technická univerzita v Košiciach", "1040 Katedra elektroniky a multimediálnych telekomunikácií") - old = row("Stará", author="Galajda, Pavol", + old = row("Stará", author="Galajda, Pavol", year=2012, school="Technická univerzita v Košiciach 1040 Katedra elektroniky a multimediálnych telekomunikácií") - new = row("Nová", author="Galajda, Pavol") + new = row("Nová", author="Galajda, Pavol", kind="Zverejnená diplomová práca", year=2014) crzp["result"] = found([old, new]) result = A._answer("Ktoré práce na KEMT napísal Pavol Galajda?", complete=crzp["no_model"]) assert "**Stará**" in result["answer"] and "**Nová**" in result["answer"] @@ -486,7 +490,7 @@ def test_school_count_uses_the_crzp_total_only_when_every_read_row_is_from_that_ dp = dict(kind="Zverejnená diplomová práca", year=2024) crzp["result"] = found([row("A", **dp), row("B", **dp)], total=1169) result = A._answer("Koľko diplomových prác bolo na TUKE v roku 2024?", complete=crzp["no_model"]) - assert "**1169** (súhrn CRZP pre hľadanie názvu školy" in result["answer"] + assert "**1169** (súhrnný počet z CRZP)" in result["answer"] and not result["truncated"] crzp["result"] = found([row("A", **dp), row("B", school="Iná škola", **dp)], total=1169) A._cache.clear() mixed = A._answer("Koľko diplomových prác bolo na TUKE v roku 2024?", complete=crzp["no_model"]) @@ -645,7 +649,7 @@ def test_old_department_name_deep_reads_and_rule_phrases(crzp): r["sid"] = key[:6] result = A._answer("Koľko prác bolo na KEMT v roku 2012?", complete=crzp["no_model"]) assert "**2**" in result["answer"] and "Hľadané výrazy" not in result["answer"] - assert {f["depth"] for _, f in crzp["searches"]} == {fetch.DEEP_ROWS} # counts read deeper than 200 rows + assert {f["depth"] for _, f in crzp["searches"]} == {20} # counted by CRZP: first page and facets only query = understand("Ktoré práce sa zaoberali rozpoznávaním reči?", use_model=False) assert query.phrases[0].endswith(" reči") and A.search_groups(query)[0][0] == query.phrases[0] from tools.crzp.answering.understand import topic_phrases @@ -690,4 +694,72 @@ def test_which_works_dealt_with_x_is_a_list_and_one_work_is_summarized_once(crzp crzp["result"] = found([row("Moja práca", author="Pták, Ján"), row("Moja práca", author="Pták, Ján")]) crzp["detail"] = parse_detail(DETAIL) result = A._answer("O čom je práca Jána Ptáka?", complete=crzp["no_model"]) - assert result["answer"].count("Zhrnutie podľa abstraktu") + result["answer"].count("Abstrakt:") == 1 + assert result["answer"].count("**Moja práca**") == 1 + + +def test_two_bachelor_theses_of_one_name_are_two_students_and_a_continuation_is_hinted(): + rows = found([row("A", author="Novák, Ján", year=2012), row("B", author="Novák, Ján", year=2020), + row("C", author="Novák, Ján", kind="Zverejnená diplomová práca", year=2014), + row("D", author="Novák, Ján", kind="Zverejnená diplomová práca", year=2022, + school="Žilinská univerzita v Žiline")])["rows"] + groups = A.identities(rows, Person(name="Ján Novák", role="author")) + tuke = [g for g in groups if g["school"] == "Technická univerzita v Košiciach"] + assert sorted(sorted(r["title"] for r in g["rows"]) for g in tuke) == [["A", "C"], ["B"]] + zilina = next(g for g in groups if g["school"] == "Žilinská univerzita v Žiline") + assert zilina["hint"] == "možno tá istá osoba ako Technická univerzita v Košiciach (2020)" + + +def test_learned_workplace_names_and_acronyms(monkeypatch): + from tools.crzp.answering import workplaces + assert workplaces.acronym("Katedra počítačov a informatiky") == "KPI" + assert workplaces._department("1040 Katedra počítačov a informatiky") == "Katedra počítačov a informatiky" + assert workplaces._department("1040 104009") is None + def search(text, **kw): + if text == "104009": + return {"facets": {"Školiteľ": [{"label": "Korečko, Štefan", "count": 9}, {"label": "Nový, Adam", "count": 5}, + {"label": "Malý, Jozef", "count": 4}, {"label": "Iný, Peter", "count": 3}]}, + "rows": []} + unit = {"Štefan Korečko": "1040 Katedra počítačov a informatiky", "Adam Nový": "1040 Katedra počítačov a informatiky", + "Jozef Malý": "1040 Katedra počítačov a inform", "Peter Iný": "1040 Katedra matematiky"}[text] + return {"facets": {}, "rows": [{"school": "Technická univerzita v Košiciach", "school_unit": unit, + "supervisors": [text.split()[1] + ", " + text.split()[0]], "title": "x"}]} + outcome = workplaces.learn_code("104009", search=search) + assert outcome["accepted"] and outcome["name"] == "Katedra počítačov a informatiky" and outcome["voters"] == 4 + assert outcome["votes"]["Katedra počítačov a informatiky"] == 3 # the cut spelling counts as the full name + + +def test_an_incomplete_wiki_profile_is_supplemented_from_crzp_for_that_person_only(crzp, monkeypatch): + from tools.crzp.answering import wiki_bridge + crzp["result"] = found([row("Moja práca", author="Pták, Ján", opponents=("Hládek, Daniel",)), + row("Cudzia", author="Pták, Ján", school="Iná univerzita v Trnave")]) + monkeypatch.setattr(wiki_bridge, "wiki_person", lambda q, db_file=None: "Ján Pták") + wiki = {"status": "no_answer", "reason": "missing_evidence", "accepted_for_delivery": True, + "answer": "V ZP Wiki som oponenta nenašiel.", "source_urls": [], "claims": [], "index_version": {"version": "v"}} + question = "Kto bol oponent bakalárskej práce Jána Ptáka?" + result = wiki_bridge.supplement(question, wiki, "rag", complete=crzp["no_model"]) + assert result["status"] == "ok" and result["answer"].startswith("V ZP Wiki som oponenta nenašiel.") + assert "**Doplnené z CRZP**" in result["answer"] and "Daniel Hládek" in result["answer"] + assert "Cudzia" not in result["answer"] # only Technická univerzita v Košiciach + import packages.claude.client as client + client.validate_metadata(result) + assert client.validated_delivery_check(result, question, "rag") + assert wiki_bridge.supplement("Aké výsledky dosiahol Ján Pták?", wiki, "rag", complete=crzp["no_model"]) is None + assert wiki_bridge.supplement("Obhájil Ján Pták bakalársku prácu?", wiki, "rag", complete=crzp["no_model"]) is None + monkeypatch.setattr(wiki_bridge, "wiki_person", lambda q, db_file=None: None) + assert wiki_bridge.supplement(question, wiki, "rag", complete=crzp["no_model"]) is None # not a wiki person + + +def test_rule_mode_uses_the_tag_crzp_records_carry(tmp_path, monkeypatch): + from tools.crzp.answering import lexicon + store = CrzpStore(tmp_path / "c.sqlite") + store.put_search("x", {"query": "x", "total": 2, "facets": {}, "complete": True, "links": True, "rows": [ + {"title": "A", "keywords": ["rozpoznávanie emócií", "reč"]}, {"title": "B", "keywords": ["Rozpoznávanie emócií"]}]}) + monkeypatch.setattr(fetch, "store", lambda: store) + monkeypatch.setitem(lexicon._state, "built", 0.0) + assert lexicon.tag_for(["rozpoznávaním", "emócií"]) == "rozpoznávanie emócií" + assert lexicon.tag_for(["rozpoznávaním", "tváre"]) is None + monkeypatch.setitem(lexicon._state, "built", 0.0) + + +def test_a_code_without_learned_evidence_stays_a_number(): + assert units.unit_label({"school": "Technická univerzita v Košiciach", "school_unit": "1040 104003"}) == "FEI, pracovisko 104003" diff --git a/tools/knowledge/academic.py b/tools/knowledge/academic.py index 68895e0..10bda90 100644 --- a/tools/knowledge/academic.py +++ b/tools/knowledge/academic.py @@ -491,16 +491,19 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, notes=[r for r in notes if set(plan.spec.facets)&set(facets_for(r['text'], {'assignment':'assignment','planned':'task','result':'progress','progress':'progress'}.get(r['phase'],'other'), plan.spec.question,r['phase']))] - notes.sort( - key=lambda r: ( - -unit_score( - r["text"], - "progress" if r["phase"] in {"result", "progress"} else "task", - ), - r["phase"] != "result", - r["line"], + if plan and plan.spec.facets: + notes.sort( + key=lambda r: ( + -unit_score( + r["text"], + "progress" if r["phase"] in {"result", "progress"} else "task", + ), + r["phase"] != "result", + r["line"], + ) ) - ) + else: # the whole meeting: its status and then its tasks, in the order of the record + notes.sort(key=lambda r: (r["phase"] == "planned", r["line"])) actual = [r for r in notes if r["phase"] in {"result", "progress"}] planned = [r for r in notes if r["phase"] == "planned"] selected = actual[:4] + planned[:4] @@ -528,7 +531,7 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None, phase = { "result": "Zaznamenaný výsledok", "progress": "Priebežný stav", - "planned": "Plán", + "planned": "Úloha zo stretnutia", "assignment": "Zadanie", }.get(note["phase"], "Záznam") quote = re.sub(r"^\s*(?:[-*+]|\d+[.)])\s+", "", note["text"]) diff --git a/tools/knowledge/planned_answers.py b/tools/knowledge/planned_answers.py index ac5c8c7..ce7fca5 100644 --- a/tools/knowledge/planned_answers.py +++ b/tools/knowledge/planned_answers.py @@ -9,6 +9,56 @@ from tools.answers.content_selection import facets_for, unit_score from tools.knowledge.evidence import claim_eligible +WORK_IN = {'diploma_thesis':'vo svojej diplomovej práci','diploma_project':'vo svojom diplomovom projekte', + 'bachelor_thesis':'vo svojej bakalárskej práci','bachelor_project':'vo svojom bakalárskom projekte'} +NOT_TOOLS = {'zpwiki','zp wiki','kemt','tuke','fei','hpc','lr','sk','hf','eval','loss','dp','bp','ok'} +TOOL_NOUNS = {'server','model','modely','modelu','dataset','datasety','databaza','databazy','framework','kniznica', + 'collection','api','agent','bot','chatbot','kontajner','sluzba','rozhranie','tools','tool'} + + +def _tool_concept(concept): + """"MCP server", "Fast120 (Qwen3.5)", "AYA collection dataset" — not "prístup na HPC" or a URL.""" + from tools.answers.content_selection import named_tools + if re.search(r'https?:|/|\.[a-z]{2,4}\b',concept) or len(concept.split())>4:return False + tools={normalized(t) for t in named_tools(concept)}-NOT_TOOLS + words=[w for w in normalized(concept).split()] + return bool(tools) and all(w in tools or w in TOOL_NOUNS or any(w in normalized(t).split() for t in tools) + for w in words) +METHOD_PHRASE = re.compile(r'(?i)\b(?:vektorov\w*\s+a\s+textov\w*|vektorov\w*|textov\w*|sémantick\w*|full-?text\w*)\s+vyhľadáv\w*' + r'|\b(?:dotrénovani\w*|fine-?tuning\w*|znalostn\w* graf\w*|grafov\w* databáz\w*)') + + +def methods_sentence(person, label, rows, candidates): + """"Ján Pták vo svojej diplomovej práci „Agent…“ využíva: RAG, MCP server, …" from the + tools and methods the cited records name (concepts of those records, named tools, method phrases).""" + from tools.answers.content_selection import named_tools + found=[];starting=set() + for row in candidates: + text=re.sub(r'https?://\S+','',re.sub(r'^\s*(?:[-*+]\s+|\d+[.)]\s+)','',row['text'])) + begins=bool(re.search(r'\b(?:zacin\w*|zacal\w*|skusa\w*|vyskusa\w*|testuj\w*)\b',normalized(text))) + before=len(found) + for phrase in METHOD_PHRASE.findall(text):found.append(phrase.strip()) + for concept in row.get('concepts') or []: + if isinstance(concept,str) and (_tool_concept(concept) or METHOD_PHRASE.fullmatch(concept.strip())): + found.append(concept.strip()) + found+=[t for t in named_tools(text) if not re.search(r'\.[a-z]{2,4}$|/',t) and normalized(t) not in NOT_TOOLS + and (len(t)>=3 or t.lower() in NOT_TOOLS)] + if begins:starting|={normalized(t) for t in found[before:]} # "začíname pracovať na GraphRAG" is no use yet + tools=[] + for name in found: + key=normalized(name) + if not key or key in NOT_TOOLS or any(f' {key} ' in f' {normalized(t)} ' for t in tools):continue + tools=[t for t in tools if f' {normalized(t)} ' not in f' {key} '] # "MCP" inside "MCP server" + tools.append(name) + if not tools:return None + topic=next((re.sub(r'^\s*(?:Téma|Názov)\s*:\s*','',r['text']).strip() for r in rows if r['kind']=='topic' + and not r['text'].strip().endswith(':') and not r['text'].lstrip().startswith('#')),None) + kind=next((r.get('work_type') for r in rows if r.get('work_type')),None) + where=WORK_IN.get(kind,'vo svojej práci')+' ('+label.split()[-1]+')' if not topic else WORK_IN.get(kind,'vo svojej práci')+' „'+topic+'“' + tools=[t+(' (začína)' if normalized(t) in starting else '') for t in tools] + return person+' '+where+' podľa záznamov využíva: '+', '.join(tools)+'.' + + def answer_facets(bound, index, units, *, engine, limit): plan=bound.spec people=[p for p in plan.people if p.role=='student'] @@ -103,6 +153,13 @@ def answer_facets(bound, index, units, *, engine, limit): distinct={} for row in candidates:distinct.setdefault(normalized(row['text']),row) candidates=list(distinct.values()) + if facet=='methods' and personal_facets[person]==['methods'] and candidates and plan.presentation not in {'comparison','table'}: + sentence=methods_sentence(person,label,groups[key],candidates) + if sentence: + lines.append(sentence);cells[(key,facet)].append(sentence) + for row in candidates: + if row['id'] not in used:evidence.append(row);used.add(row['id']) + continue lines.append(FACET_LABELS[facet]+':') if not candidates: notice=person+' — '+label+': '+FACET_LABELS[facet].lower()+': v podkladoch chýbajú doložené údaje.' diff --git a/tools/knowledge/roster.py b/tools/knowledge/roster.py index 996426e..d7888dc 100644 --- a/tools/knowledge/roster.py +++ b/tools/knowledge/roster.py @@ -281,9 +281,12 @@ def profile_tags(db_file=None) -> dict: def answer_related(spec, index, *, engine: str, db_file=None) -> dict: from collections import Counter asked = related_topic(spec) + from tools.retrieval.synonyms import variants_pattern, base_name + # "súvisia s rozpoznávaním reči": every case form of the topic and of its reviewed + # synonyms ("speech recognition", "ASR"), not only the literal. stem = topic_stem(asked) - # "súvisia s chatbotom": every case form of the topic, not only the literal. - word = re.compile(r"(? dict: # of the matching tag ("chatbotom" -> "chatbot"). spelled = original_phrase(spec.question, asked) topic = spelled if normalized(spelled) == key else key + if base_name(asked) != asked: + topic = base_name(asked) # "rozpoznávaním reči" -> "rozpoznávanie reči" units = [r for r in index["units"] if claim_eligible(r) and r.get("work_label") and not administrative_only(r["text"])] mentions = [r for r in units if word.search(normalized(r["text"])) or any(word.fullmatch(normalized(c)) for c in r.get("concepts", []))] diff --git a/tools/retrieval/synonyms.py b/tools/retrieval/synonyms.py new file mode 100644 index 0000000..f7af4d5 --- /dev/null +++ b/tools/retrieval/synonyms.py @@ -0,0 +1,83 @@ +"""Reviewed topic synonyms shared by ZP Wiki (RAG, GraphRAG) and CRZP. + +One group is one meaning; the first entry is the base Slovak name shown in +answers. Matching ignores case, diacritics and Slovak case endings +(phrase_pattern), so "rozpoznávaním reči", "speech recognition" and "ASR" +all find the same topic. Groups are reviewed by hand; no synonym is guessed. +""" +from __future__ import annotations + +import re + +from tools.answers.context_policy import normalized + +GROUPS = [ + ["rozpoznávanie reči", "speech recognition", "ASR", "automatické rozpoznávanie reči", "prepis reči", "speech to text"], + ["syntéza reči", "speech synthesis", "text to speech", "TTS"], + ["chatbot", "dialógový systém", "konverzačný agent", "conversational agent"], + ["veľký jazykový model", "LLM", "large language model", "jazykový model", "language model"], + ["RAG", "retrieval augmented generation", "vyhľadávanie s generovaním"], + ["znalostný graf", "knowledge graph"], + ["GraphRAG", "graph RAG"], + ["strojový preklad", "machine translation", "neurónový strojový preklad"], + ["sumarizácia", "summarization", "sumarizácia textu", "zhrnutie textu"], + ["rozpoznávanie pomenovaných entít", "NER", "named entity recognition"], + ["analýza sentimentu", "sentiment analysis"], + ["nenávistná reč", "hate speech", "toxický jazyk", "toxicita", "toxic language"], + ["neurónová sieť", "neural network", "neurónové siete"], + ["hlboké učenie", "deep learning"], + ["strojové učenie", "machine learning"], + ["umelá inteligencia", "artificial intelligence", "AI"], + ["spracovanie prirodzeného jazyka", "NLP", "natural language processing"], + ["počítačové videnie", "computer vision"], + ["internet vecí", "IoT", "internet of things"], + ["kybernetická bezpečnosť", "cybersecurity", "cyber security"], + ["vektorové vyhľadávanie", "vector search", "sémantické vyhľadávanie", "semantic search"], + ["oprava gramatiky", "grammatical error correction", "korekcia textu", "oprava interpunkcie"], + ["generovanie otázok", "question generation"], + ["rozpoznávanie emócií", "emotion recognition"], +] +_ENDING = r"\w{0,4}" + + +def _key(text: str) -> str: + return " ".join(normalized(text).split()) + + +def _stem(word: str) -> str: + """"rozpoznavanim" -> "rozpoznavan"; short words and acronyms stay whole.""" + if len(word) < 5 or not word.isalpha(): + return word + for ending in ("ami", "ach", "och", "ovi", "ych", "ymi", "eho", "emu", "iou", "ou", "om", "ov", "ej", "ia", "ie", + "iu", "im", "y", "a", "e", "i", "u", "o"): + if word.endswith(ending) and len(word) - len(ending) >= 4: + return word[: -len(ending)] + return word + + +def phrase_pattern(text: str) -> str: + """A regex (over normalized text) for the phrase in any Slovak case form.""" + words = _key(text).split() + if not words: + return r"(?!)" + parts = [re.escape(_stem(w)) + (_ENDING if len(w) >= 5 and w.isalpha() else "") for w in words] + return r"(? list[str]: + """The reviewed group the phrase belongs to (any case form), else [text].""" + key = _key(text) + for group in GROUPS: + if any(re.fullmatch(phrase_pattern(item), key) for item in group): + return group + return [text] + + +def base_name(text: str) -> str: + group = group_of(text) + return group[0] if len(group) > 1 else text + + +def variants_pattern(text: str) -> re.Pattern: + """One regex matching the topic and its reviewed synonyms in any case form.""" + return re.compile("|".join(phrase_pattern(item) for item in group_of(text)))