jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

upravy vyhladavania

Browse Source
This commit is contained in:
Ján Pták 2026-11-15 16:27:08 +01:00
parent 64bde676c3
commit e8fed0a960
50 changed files with 2235 additions and 662 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/routes.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import json import json
import logging import logging
import os
import re import re
import sqlite3 import sqlite3
from typing import ( from typing import (
@ -171,7 +172,7 @@ def verified_answer(request: AnswerRequest) -> dict:
if request.engine == "crzp": if request.engine == "crzp":
from tools.crzp.answering.answer import answer as crzp_answer from tools.crzp.answering.answer import answer as crzp_answer
result = crzp_answer(request.query, limit=request.limit, context_token=request.context_token) result = crzp_answer(request.query, limit=request.limit, context_token=request.context_token)
else: elif request.include_diagnostics:
result = deliver_answer( result = deliver_answer(
request.query, request.query,
engine=request.engine, engine=request.engine,
@ -179,6 +180,10 @@ def verified_answer(request: AnswerRequest) -> dict:
include_diagnostics=request.include_diagnostics, include_diagnostics=request.include_diagnostics,
**({'context_token':request.context_token} if request.context_token else {}), **({'context_token':request.context_token} if request.context_token else {}),
) )
else: # ZP Wiki and CRZP at the same time, one answer (tools/answers/combined.py)
from tools.answers.combined import answer as combined_answer
result = combined_answer(request.query, engine=request.engine, limit=request.limit,
context_token=request.context_token, deliver=deliver_answer)
except Exception: except Exception:
logger.exception("Príprava overenej odpovede zlyhala.") logger.exception("Príprava overenej odpovede zlyhala.")
return { return {

2
conftest.py
View File

@ -11,6 +11,8 @@ os.environ.setdefault("RETRIEVAL_RERANKER", "off")
# No background CRZP traffic from tests that start the API. # No background CRZP traffic from tests that start the API.
os.environ.setdefault("CRZP_CHECK_INTERVAL_SECONDS", "0") os.environ.setdefault("CRZP_CHECK_INTERVAL_SECONDS", "0")
os.environ.setdefault("CRZP_PREPARE", "0") os.environ.setdefault("CRZP_PREPARE", "0")
os.environ.setdefault("ANSWER_WITH_CRZP", "0") # tests answer from the wiki fixture only
os.environ.setdefault("CRZP_GRAPH_LINKS", "0")
# Evaluation code lives outside Git (see local/OBSAH.md). Without it, tests that # Evaluation code lives outside Git (see local/OBSAH.md). Without it, tests that
# import it at module level are skipped at collection instead of failing. # import it at module level are skipped at collection instead of failing.

3
deployment/docker-compose.yml
View File

@ -38,6 +38,9 @@ services:
RETRIEVAL_LEMMATIZER: ${RETRIEVAL_LEMMATIZER:-stanza} RETRIEVAL_LEMMATIZER: ${RETRIEVAL_LEMMATIZER:-stanza}
RETRIEVAL_RERANKER: ${RETRIEVAL_RERANKER:-on} RETRIEVAL_RERANKER: ${RETRIEVAL_RERANKER:-on}
ANSWER_QUESTION_CATALOG: ${ANSWER_QUESTION_CATALOG:-1} ANSWER_QUESTION_CATALOG: ${ANSWER_QUESTION_CATALOG:-1}
# Few glibc malloc arenas: many short worker threads (wiki + CRZP in parallel) otherwise keep
# a separate arena each and the process grows by ~100 MB per question on a 6 GB machine.
MALLOC_ARENA_MAX: "2"
GIT_SSH_COMMAND: >- GIT_SSH_COMMAND: >-
ssh -F /dev/null -i /run/secrets/zpwiki_ssh_key ssh -F /dev/null -i /run/secrets/zpwiki_ssh_key
-o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes

7
packages/claude/client.py
View File

@ -16,7 +16,7 @@ import urllib.parse
import urllib.request import urllib.request
MAX_RESPONSE_BYTES = 2 * 1024 * 1024 MAX_RESPONSE_BYTES = 2 * 1024 * 1024
ENGINES = ("rag", "graphrag", "crzp") ENGINES = ("rag", "graphrag") # CRZP is part of both answers
STATUSES = {"ok", "no_answer", "ambiguous", "error"} STATUSES = {"ok", "no_answer", "ambiguous", "error"}
@ -103,7 +103,7 @@ def error_result(reason):
"forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.",
"invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.",
"question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.",
"engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.",
"crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.",
"model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.",
'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.',
@ -217,14 +217,13 @@ def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=Non
if ( if (
engine not in ENGINES engine not in ENGINES
or mode not in {"answer", "retrieve"} or mode not in {"answer", "retrieve"}
or engine == "crzp" and mode != "answer"
or type(limit) is not int or type(limit) is not int
or not 1 <= limit <= 20 or not 1 <= limit <= 20
or not isinstance(query, str) or not isinstance(query, str)
): ):
raise ClientError("invalid_query") raise ClientError("invalid_query")
query = query.strip() query = query.strip()
command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) command = re.match(r"^/(rag|graphrag)(?=\s|$)", query, re.I)
if command: if command:
if command[1].lower() != engine: if command[1].lower() != engine:
raise ClientError("engine_mismatch") raise ClientError("engine_mismatch")

15
packages/claude/dist/crzp/manifest.json vendored
View File

@ -1,15 +0,0 @@
{
"backend_required": true,
"default_mode": "verified_answer",
"engine": "crzp",
"entrypoint": "SKILL.md",
"files": {
"SKILL.md": "aa97aeecb3ac09e06a5bdb1e613d7ba67674b36349effc4220ffde98a506b77a",
"scripts/client.py": "13c7eb8ae62a2cec5b729518422eb4d102eb197da7cb4bc3aa521d1940ea6812",
"scripts/query.py": "642fd4646889caa839c2700883bbbe67f6d6edfcff7474780c8b29eeda200d19"
},
"minimum_python": "3.10",
"name": "crzp",
"schema_version": 1,
"version": "1.13.0"
}

444
packages/claude/dist/crzp/scripts/client.py vendored
View File

@ -1,444 +0,0 @@
"""Small, dependency-free client shared by Claude skills and the MCP bridge."""
from __future__ import annotations
import argparse
import hashlib
from dataclasses import dataclass
import ipaddress
import json
import os
import re
from pathlib import Path
import stat
import urllib.error
import urllib.parse
import urllib.request
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
ENGINES = ("rag", "graphrag", "crzp")
STATUSES = {"ok", "no_answer", "ambiguous", "error"}
class ClientError(Exception):
"""The message is a public reason code, never an exception from the transport."""
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
@dataclass(frozen=True)
class Config:
url: str
key: str
timeout: float = 90
allow_http: bool = False
@classmethod
def load(cls):
values = {}
filename = os.getenv("ZP_AGENT_CONFIG", "").strip()
if filename:
try:
path = Path(filename).expanduser()
if os.name == "posix" and stat.S_IMODE(path.stat().st_mode) & 0o077:
raise ClientError("config_permissions_require_0600")
values = json.loads(path.read_text())
if not isinstance(values, dict):
raise ValueError()
except ClientError:
raise
except (OSError, ValueError):
raise ClientError("invalid_configuration") from None
try:
config = cls(
os.getenv("ZP_AGENT_URL", values.get("backend_url", "")).strip().rstrip("/"),
os.getenv("ZP_AGENT_API_KEY", values.get("api_key", "")).strip(),
float(os.getenv("ZP_AGENT_TIMEOUT", values.get("timeout_seconds", 90))),
os.getenv("ZP_AGENT_ALLOW_HTTP", "").lower() in {"1", "true"},
)
config.validate()
return config
except (ValueError, TypeError, AttributeError):
raise ClientError("invalid_configuration") from None
def validate(self):
try:
url = urllib.parse.urlsplit(self.url)
if (
url.scheme not in {"http", "https"}
or not url.hostname
or url.username is not None
or url.password is not None
or url.query
or url.fragment
or any(c.isspace() for c in self.url)
):
raise ValueError()
url.port
if url.scheme == "http":
local = url.hostname == "localhost"
try:
local = local or ipaddress.ip_address(url.hostname).is_loopback
except ValueError:
pass
if not local and not self.allow_http:
raise ValueError()
if not self.key or "\r" in self.key or "\n" in self.key or not 1 <= self.timeout <= 600:
raise ValueError()
except (ValueError, TypeError):
raise ClientError("invalid_configuration") from None
def error_result(reason):
messages = {
'invalid_context':'Kontext predchádzajúcej otázky sa nepodarilo overiť. Zopakuj úplnú otázku s osobou, prácou a prípadným dátumom.',
'stale_context':'Dáta alebo kontext konverzácie sa zmenili. Zopakuj úplnú otázku; starý výber stretnutia nepoužijem.',
'expired_context':'Kontext predchádzajúcej otázky už vypršal. Zopakuj úplnú otázku.',
'proposal_not_available':'Otázka žiada návrh možného prepojenia, nie iba existujúce fakty. Návrhový režim ešte nie je dostupný; výpis podkladov preto nevydávam za hotový návrh.',
"reference_unavailable": "Zdrojový repozitár sa nepodarilo verejne overiť. Odpoveď založenú iba na tomto odkaze neuvádzam.",
"unauthorized": "Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.",
"forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.",
"invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.",
"question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.",
"engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.",
"crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.",
"model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.",
'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.',
"answer_rejected": "Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. "
"Neoverené údaje sa nezobrazia.",
"answer_unavailable": "Overenú odpoveď sa momentálne nepodarilo pripraviť. Skús to neskôr.",
"answer_capacity_exceeded": "Agent práve spracúva iné otázky. Skús to o chvíľu.",
"backend_busy": "Agent práve spracúva iné otázky. Skús to o chvíľu.",
"backend_timeout": "Vyhľadávanie neodpovedalo včas. Skús otázku zopakovať neskôr.",
"backend_unavailable": "Spojenie s backendom momentálne nie je dostupné. Skontroluj, či bežia služby a tunel.",
"retrieval_unavailable": "Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.",
"content_index_unavailable": "Obsahové vyhľadávanie momentálne nie je pripravené alebo sa jeho výstup "
"nepodarilo overiť. Treba overiť extrakciu pre aktuálnu verziu dát; "
"nejde o potvrdenie, že hľadané údaje neexistujú.",
}
return {
"status": "error",
"reason": reason,
"accepted_for_delivery": False,
"answer": messages.get(reason, "Overenú odpoveď sa momentálne nepodarilo získať."),
"sources": [],
"index_version": None,
"complete": False,
"truncated": False,
"truncation_reasons": [],
}
def request_json(config: Config, endpoint: str, body=None):
config.validate()
request = urllib.request.Request(
config.url + endpoint,
data=json.dumps(body, ensure_ascii=False).encode() if body is not None else None,
headers={
"Authorization": "Bearer " + config.key,
"Accept": "application/json",
**({"Content-Type": "application/json"} if body is not None else {}),
},
)
try:
with urllib.request.build_opener(NoRedirect()).open(
request, timeout=config.timeout
) as response:
raw = response.read(MAX_RESPONSE_BYTES + 1)
except urllib.error.HTTPError as exc:
code = exc.code
exc.close()
reason = {
401: "unauthorized",
403: "forbidden",
404: "backend_endpoint_not_found",
400: "invalid_query",
422: "invalid_request",
429: "backend_busy",
503: "backend_unavailable",
}.get(code, "backend_http_error")
raise ClientError("redirect_not_allowed" if 300 <= code < 400 else reason) from None
except (urllib.error.URLError, TimeoutError, OSError) as exc:
timeout = isinstance(exc, TimeoutError) or isinstance(
getattr(exc, "reason", None), TimeoutError
)
raise ClientError("backend_timeout" if timeout else "backend_unavailable") from None
if len(raw) > MAX_RESPONSE_BYTES:
raise ClientError("response_too_large")
try:
value = json.loads(raw)
if not isinstance(value, dict):
raise ValueError()
return value
except (ValueError, UnicodeError):
raise ClientError("invalid_backend_response") from None
def validate_metadata(value):
if (
not {"complete", "truncated", "truncation_reasons", "index_version"} <= value.keys()
or value.get("complete") is not None
and type(value["complete"]) is not bool
or type(value.get("truncated")) is not bool
or value["truncated"]
and value.get("complete") is not False
or not isinstance(value.get("truncation_reasons"), list)
or not all(isinstance(x, str) for x in value["truncation_reasons"])
or value["truncation_reasons"]
and not value["truncated"]
or value.get("index_version") is not None
and not isinstance(value["index_version"], dict)
):
raise ClientError("invalid_backend_response")
def validated_delivery_check(value,query,engine):
check=value.get('delivery_check')
if check is None:return None
digest=hashlib.sha256(json.dumps([engine,query.strip()],ensure_ascii=False,separators=(',',':')).encode()).hexdigest()
if (not isinstance(check,dict) or set(check)!={'layer','code','request_fingerprint'}
or check.get('request_fingerprint')!=digest
or check.get('layer') not in {'delivery','scope','evidence','coverage','answer','verification','fulfilment','context','planning','retrieval','service','evidence_selection'}
or check.get('code') not in {'complete','partial','not_applicable','wrong_person','wrong_work_or_meeting',
'wrong_period','mixed_snapshot','altered_evidence','missing_evidence','missing_person','invalid_citation','verification_unavailable',
'request_not_fulfilled','invalid_review','undisclosed_gap','clarification_required','evidence_not_found','unavailable',
'invalid_model_selection','model_unavailable'}):
raise ClientError('invalid_backend_response')
return check
def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=None, context_token=None):
"""Return only approved answer fields, or explicitly marked retrieval context."""
checked_delivery=None
try:
if (
engine not in ENGINES
or mode not in {"answer", "retrieve"}
or engine == "crzp" and mode != "answer"
or type(limit) is not int
or not 1 <= limit <= 20
or not isinstance(query, str)
):
raise ClientError("invalid_query")
query = query.strip()
command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I)
if command:
if command[1].lower() != engine:
raise ClientError("engine_mismatch")
query = query[command.end() :].strip()
if not 1 <= len(query) <= 500:
raise ClientError("invalid_query")
config = config or Config.load()
if context_token is not None and (mode!='answer' or not isinstance(context_token,str) or not 1<=len(context_token)<=16000):
raise ClientError('invalid_query')
endpoint = "/answer" if mode == "answer" else "/" + engine
body = {"query": query.strip(), "limit": limit}
if context_token is not None:body['context_token']=context_token
if mode == "answer" or engine == "rag":
body["include_diagnostics"] = False
if mode == "answer":
body["engine"] = engine
else:
body["published_only"] = True
value = request_json(config, endpoint, body)
if mode=='answer':checked_delivery=validated_delivery_check(value,query,engine)
if not isinstance(value.get("status"), str) or value["status"] not in STATUSES:
raise ClientError("invalid_backend_response")
if value["status"] == "error":
# Raw model output, diagnostics and rejected text must not escape.
reason = value.get("reason")
safe_reasons = {
'verification_unavailable',
'invalid_context','stale_context','expired_context','proposal_not_available',
"model_unavailable",
"answer_capacity_exceeded",
"answer_rejected",
"answer_unavailable",
"retrieval_unavailable",
"content_index_unavailable",
"reference_unavailable",
"crzp_unavailable",
}
raise ClientError(
reason
if isinstance(reason, str) and reason in safe_reasons
else "backend_answer_error" if mode == "answer" else "retrieval_unavailable"
)
validate_metadata(value)
fields = {
"status",
"reason",
"index_version",
"complete",
"truncated",
"truncation_reasons",
}
if mode == "answer":
if not isinstance(value.get("query"), str):
raise ClientError("invalid_backend_response")
if value["query"] != body["query"] or value.get("engine") != engine:
raise ClientError("question_mismatch")
if value.get("accepted_for_delivery") is not True:
raise ClientError("answer_rejected")
if (
not isinstance(value.get("answer"), str)
or not value["answer"].strip()
or not isinstance(value.get("source_urls"), list)
or not all(isinstance(x, str) for x in value["source_urls"])
or not isinstance(value.get("claims"), list)
):
raise ClientError("invalid_backend_response")
if "answer_coverage" in value:
coverage = value["answer_coverage"]
if (
not isinstance(coverage, dict)
or coverage.get("scope")
not in ("fact", "overview", "assignment", "list", "count")
or coverage.get("status")
not in ("supported", "partial", "unknown", "not_found", "unavailable")
or any(
not isinstance(coverage.get(key), list)
or any(not isinstance(item, str) for item in coverage[key])
for key in ["warnings", "limitations"]
)
):
raise ClientError("invalid_backend_response")
if "knowledge_version" in value and not isinstance(value["knowledge_version"], str):
raise ClientError("invalid_backend_response")
if 'conversation_context' in value:
context=value['conversation_context']
if (not isinstance(context,dict) or not isinstance(context.get('token'),str)
or not 1<=len(context['token'])<=16000
or any(not isinstance(context.get(k),list) for k in ['work_options','meeting_options'])):
raise ClientError('invalid_backend_response')
if 'resolved_query' in value and not isinstance(value['resolved_query'],str):
raise ClientError('invalid_backend_response')
if 'answer_kind' in value and value['answer_kind'] not in {'grounded_facts','recommendation'}:
raise ClientError('invalid_backend_response')
if 'recommendation_status' in value and value['recommendation_status'] not in {'accepted','insufficient_grounding','unavailable'}:
raise ClientError('invalid_backend_response')
fields |= {
"query",
"answer",
"source_urls",
"claims",
"count",
"accepted_for_delivery",
"answer_coverage",
}
fields.add("knowledge_version")
fields |= {'conversation_context','resolved_query','answer_kind','recommendation_status','delivery_check'}
else:
if not isinstance(value.get("sources"), list):
raise ClientError("invalid_backend_response")
if engine == "rag" and not isinstance(value.get("context"), str):
raise ClientError("invalid_backend_response")
if engine == "graphrag" and (
not isinstance(value.get("facts"), list) or not isinstance(value.get("paths"), list)
):
raise ClientError("invalid_backend_response")
fields |= {
"sources",
"context",
"facts",
"supporting_facts",
"paths",
"aggregation",
"constraints",
"answer_candidates",
"instructions",
}
result = {key: item for key, item in value.items() if key in fields}
return {
**result,
"engine": engine,
"output_kind": "verified_answer" if mode == "answer" else "retrieval_context",
**({"requires_final_answer_verification": True} if mode == "retrieve" else {}),
}
except ClientError as exc:
return {**error_result(str(exc)),**({'delivery_check':checked_delivery} if checked_delivery else {})}
def backend_health(config=None):
try:
value = request_json(config or Config.load(), "/health")
tools = value.get("tools")
if (
type(value.get("ready")) is not bool
or not isinstance(tools, dict)
or any(
not isinstance(tools.get(engine), dict)
or type(tools[engine].get("ready")) is not bool
for engine in ("rag", "graphrag")
)
):
raise ClientError("invalid_backend_response")
return {
"status": "ok" if value["ready"] else "error",
"ready": value["ready"],
"index_version": value.get("index_version"),
**({"capabilities": value["capabilities"]} if isinstance(value.get("capabilities"), dict) else {}),
**({"answer_ready": value["answer_ready"]} if isinstance(value.get("answer_ready"), dict) else {}),
"tools": {
engine: {
k: v
for k, v in tools[engine].items()
if k in {"ready", "reason", "index_version"}
}
for engine in ENGINES
if isinstance(tools.get(engine), dict) and type(tools[engine].get("ready")) is bool
},
}
except ClientError as exc:
return {**error_result(str(exc)), "ready": False}
def answer_for_presentation(result: dict) -> dict:
"""Keep approved prose/caveats intact; retrieval telemetry is not answer prose.
Called after query_backend validates the complete backend contract. Full
metadata remains available there, in /answer and in diagnostic retrieval.
Never change an incomplete result into a claim of completeness.
"""
fields = {
'conversation_context','resolved_query','answer_kind','recommendation_status','delivery_check',
"status",
"reason",
"query",
"engine",
"output_kind",
"accepted_for_delivery",
"answer",
"source_urls",
"index_version",
"knowledge_version",
}
return {key: value for key, value in result.items() if key in fields}
def main(engine=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("query", nargs="?")
if engine is None:
parser.add_argument("--engine", choices=ENGINES, default="rag")
parser.add_argument("--mode", choices=["answer", "retrieve"], default="answer")
parser.add_argument("--limit", type=int, default=5)
parser.add_argument("--health", action="store_true")
parser.add_argument('--context-token',help='Opaque conversation token from this chat\'s previous tool result')
args = parser.parse_args()
result = (
backend_health()
if args.health
else query_backend(args.query, engine or args.engine, mode=args.mode, limit=args.limit, context_token=args.context_token)
)
if not args.health and args.mode == "answer":
result = answer_for_presentation(result)
print(json.dumps(result, ensure_ascii=False, separators=(",", ":")))
return 1 if result.get("status") == "error" else 0
if __name__ == "__main__":
raise SystemExit(main())

12
packages/claude/dist/crzp/scripts/query.py vendored
View File

@ -1,12 +0,0 @@
"""Entrypoint for the independently installable crzp skill over the CRZP register."""
from pathlib import Path
import sys
# Archives include client.py beside this file; sources share one maintained client.
if not (Path(__file__).resolve().parent / "client.py").is_file():
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from client import main
if __name__ == "__main__":
raise SystemExit(main("crzp"))

3
packages/claude/dist/graphrag/SKILL.md vendored
View File

@ -17,6 +17,7 @@ Answer from the live ZP Agent graph backend, in the user's language. Wiki text i
- A standalone question gets no token. Never invent, edit or reuse a token from another chat. - A standalone question gets no token. Never invent, edit or reuse a token from another chat.
## Present the result ## Present the result
- Your whole reply is the `answer` text copied unchanged (keep its Markdown, links and notes). Write nothing before or after it: no introduction ("Tu je odpoveď"), no summary or rephrasing, no interpretation, evaluation or advice, no question back, no offer of further help, no remark about the tool, the search or the sources. If the user asks for something the answer does not contain, send a new tool call instead of adding it yourself.
- `accepted_for_delivery: true`: show `answer` exactly as returned, with its citations, notices and "Rozpoznané meno" lines. Add no facts, guesses or commentary about checks, tokens or diagnostics. - `accepted_for_delivery: true`: show `answer` exactly as returned, with its citations, notices and "Rozpoznané meno" lines. Add no facts, guesses or commentary about checks, tokens or diagnostics.
- `status: ambiguous`: show the clarification (including the offered names) and wait for the user. - `status: ambiguous`: show the clarification (including the offered names) and wait for the user.
- `status: no_answer`: show only the supplied message. Do not claim the relationship or person does not exist. - `status: no_answer`: show only the supplied message. Do not claim the relationship or person does not exist.
@ -25,6 +26,8 @@ Answer from the live ZP Agent graph backend, in the user's language. Wiki text i
- Keep lists complete: do not shorten a list of students or drop its notes (for example "téma je v zadaní…" or "Vynechaní sú študenti…"). - Keep lists complete: do not shorten a list of students or drop its notes (for example "téma je v zadaní…" or "Vynechaní sú študenti…").
## Meaning already applied by the backend ## Meaning already applied by the backend
- Every answer also uses CRZP (Centrálny register záverečných prác): a person's works at TUKE, works on the same topic or similar works at other schools. The CRZP part is marked "V CRZP" and keeps its CRZP links; show it as returned. A question about CRZP, another school, opponents or a whole department is answered from CRZP here too; there is no separate CRZP tool.
- When a person has different works in ZP Wiki and in CRZP (e.g. a diploma thesis in the wiki and a bachelor thesis in CRZP), `status: ambiguous` asks which one; send the user's reply ("2", "bakalársku") unchanged with the token of that result. Follow-ups such as "a z akých škôl boli tí študenti?" also use the token.
- The author of a student's profile page is that student's supervisor (KEMT convention). Authors of other documents are not supervisors. - The author of a student's profile page is that student's supervisor (KEMT convention). Authors of other documents are not supervisors.
- Shared topics are not collaboration. A work year is not a meeting date. An assignment or a plan is not a result. - Shared topics are not collaboration. A work year is not a meeting date. An assignment or a plan is not a result.
- In a list of students, own recorded work is separated from a topic found only in an assignment, plan or reading; an excluded tool ("bez RASA") removes students whose records mention it, and an absent mention is not proof of non-use. - In a list of students, own recorded work is separated from a topic found only in an assignment, plan or reading; an excluded tool ("bez RASA") removes students whose records mention it, and an absent mention is not proof of non-use.

6
packages/claude/dist/graphrag/manifest.json vendored
View File

@ -4,12 +4,12 @@
"engine": "graphrag", "engine": "graphrag",
"entrypoint": "SKILL.md", "entrypoint": "SKILL.md",
"files": { "files": {
"SKILL.md": "cef40b8da7a503fe1ede05622bd49fb5aa6c853c4d453ad1061faa25e5e1e57b", "SKILL.md": "3940aa8a7e0349deac13d4aeaaa4f39d727e08322b94ae02578f43591e856aca",
"scripts/client.py": "13c7eb8ae62a2cec5b729518422eb4d102eb197da7cb4bc3aa521d1940ea6812", "scripts/client.py": "5f75727ae6e071f6f145a3a61988ada0e716476e63d7f3c7b08786c210910f16",
"scripts/query.py": "463460cd93cf2a584156df6aa571bcf98297e5a1a7258581d02b084b1e1a6180" "scripts/query.py": "463460cd93cf2a584156df6aa571bcf98297e5a1a7258581d02b084b1e1a6180"
}, },
"minimum_python": "3.10", "minimum_python": "3.10",
"name": "graphrag", "name": "graphrag",
"schema_version": 1, "schema_version": 1,
"version": "1.13.0" "version": "2.0.0"
} }

7
packages/claude/dist/graphrag/scripts/client.py vendored
View File

@ -16,7 +16,7 @@ import urllib.parse
import urllib.request import urllib.request
MAX_RESPONSE_BYTES = 2 * 1024 * 1024 MAX_RESPONSE_BYTES = 2 * 1024 * 1024
ENGINES = ("rag", "graphrag", "crzp") ENGINES = ("rag", "graphrag") # CRZP is part of both answers
STATUSES = {"ok", "no_answer", "ambiguous", "error"} STATUSES = {"ok", "no_answer", "ambiguous", "error"}
@ -103,7 +103,7 @@ def error_result(reason):
"forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.",
"invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.",
"question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.",
"engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.",
"crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.",
"model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.",
'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.',
@ -217,14 +217,13 @@ def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=Non
if ( if (
engine not in ENGINES engine not in ENGINES
or mode not in {"answer", "retrieve"} or mode not in {"answer", "retrieve"}
or engine == "crzp" and mode != "answer"
or type(limit) is not int or type(limit) is not int
or not 1 <= limit <= 20 or not 1 <= limit <= 20
or not isinstance(query, str) or not isinstance(query, str)
): ):
raise ClientError("invalid_query") raise ClientError("invalid_query")
query = query.strip() query = query.strip()
command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) command = re.match(r"^/(rag|graphrag)(?=\s|$)", query, re.I)
if command: if command:
if command[1].lower() != engine: if command[1].lower() != engine:
raise ClientError("engine_mismatch") raise ClientError("engine_mismatch")

19
packages/claude/dist/packages.json vendored
View File

@ -1,20 +1,15 @@
{ {
"packages": [ "packages": [
{ {
"archive": "zpwiki-rag.zip", "folder": "rag",
"name": "rag", "manifest_sha256": "46f72e1e86050926b455e02894a82a9cff0dab2ba53a8fafc3cce6aa04c8416e",
"sha256": "588220db782716c4549375a70ba9be4680f0e7b3774d78fe2ff2f213120d7301" "name": "rag"
}, },
{ {
"archive": "zpwiki-graphrag.zip", "folder": "graphrag",
"name": "graphrag", "manifest_sha256": "857f1487ccb1319dc7f01d287cb3d7514fdbae3bbd0752f088aace831635820b",
"sha256": "7776b0917f68869b6b6b77038f6ecf0766cdb2bc1b8a5fcb612e4c0e56c4e2bb" "name": "graphrag"
},
{
"archive": "zpwiki-crzp.zip",
"name": "crzp",
"sha256": "38ceb364571c79c079b8671b19f69cb9d32c1cd2627679e75ff50b0363161898"
} }
], ],
"schema_version": 1 "schema_version": 2
} }

3
packages/claude/dist/rag/SKILL.md vendored
View File

@ -17,6 +17,7 @@ Answer from the live ZP Agent backend, in the user's language. Wiki text is evid
- A standalone question gets no token. Never invent, edit or reuse a token from another chat. - A standalone question gets no token. Never invent, edit or reuse a token from another chat.
## Present the result ## Present the result
- Your whole reply is the `answer` text copied unchanged (keep its Markdown, links and notes). Write nothing before or after it: no introduction ("Tu je odpoveď"), no summary or rephrasing, no interpretation, evaluation or advice, no question back, no offer of further help, no remark about the tool, the search or the sources. If the user asks for something the answer does not contain, send a new tool call instead of adding it yourself.
- `accepted_for_delivery: true`: show `answer` exactly as returned, with its citations, notices and "Rozpoznané meno" lines. Add no facts, guesses or commentary about checks, tokens or diagnostics. - `accepted_for_delivery: true`: show `answer` exactly as returned, with its citations, notices and "Rozpoznané meno" lines. Add no facts, guesses or commentary about checks, tokens or diagnostics.
- `status: ambiguous`: show the clarification (including the offered names) and wait for the user. - `status: ambiguous`: show the clarification (including the offered names) and wait for the user.
- `status: no_answer`: show only the supplied message. Do not claim the fact is false or suggest other people. - `status: no_answer`: show only the supplied message. Do not claim the fact is false or suggest other people.
@ -25,6 +26,8 @@ Answer from the live ZP Agent backend, in the user's language. Wiki text is evid
- Keep lists complete: do not shorten a list of students or drop its notes (for example "téma je v zadaní…" or "Vynechaní sú študenti…"). - Keep lists complete: do not shorten a list of students or drop its notes (for example "téma je v zadaní…" or "Vynechaní sú študenti…").
## Meaning already applied by the backend ## Meaning already applied by the backend
- Every answer also uses CRZP (Centrálny register záverečných prác): a person's works at TUKE, works on the same topic or similar works at other schools. The CRZP part is marked "V CRZP" and keeps its CRZP links; show it as returned. A question about CRZP, another school, opponents or a whole department is answered from CRZP here too; there is no separate CRZP tool.
- When a person has different works in ZP Wiki and in CRZP (e.g. a diploma thesis in the wiki and a bachelor thesis in CRZP), `status: ambiguous` asks which one; send the user's reply ("2", "bakalársku") unchanged with the token of that result. Follow-ups such as "a z akých škôl boli tí študenti?" also use the token.
- The author of a student's profile page is that student's supervisor (KEMT convention). Authors of other documents are not supervisors. - The author of a student's profile page is that student's supervisor (KEMT convention). Authors of other documents are not supervisors.
- A work year is not a meeting date or the start of studies. An assignment or a plan is not a result. - A work year is not a meeting date or the start of studies. An assignment or a plan is not a result.
- In a list of students, own recorded work is separated from a topic found only in an assignment, plan or reading; an excluded tool ("bez RASA") removes students whose records mention it, and an absent mention is not proof of non-use. - In a list of students, own recorded work is separated from a topic found only in an assignment, plan or reading; an excluded tool ("bez RASA") removes students whose records mention it, and an absent mention is not proof of non-use.

6
packages/claude/dist/rag/manifest.json vendored
View File

@ -4,12 +4,12 @@
"engine": "rag", "engine": "rag",
"entrypoint": "SKILL.md", "entrypoint": "SKILL.md",
"files": { "files": {
"SKILL.md": "5f359a7cee5021778bc64e999cd2734427fca1674120cf87685bf1a80b376807", "SKILL.md": "b64bd4b88857f0fc03bb1a3fbffc4cac8ef4ca6f00eacea4f9b4064ff3d55ee8",
"scripts/client.py": "13c7eb8ae62a2cec5b729518422eb4d102eb197da7cb4bc3aa521d1940ea6812", "scripts/client.py": "5f75727ae6e071f6f145a3a61988ada0e716476e63d7f3c7b08786c210910f16",
"scripts/query.py": "f482f2dc53430cbb383bce4fcc1cccaf580fc7b5a0c885a217fc40cf53713277" "scripts/query.py": "f482f2dc53430cbb383bce4fcc1cccaf580fc7b5a0c885a217fc40cf53713277"
}, },
"minimum_python": "3.10", "minimum_python": "3.10",
"name": "rag", "name": "rag",
"schema_version": 1, "schema_version": 1,
"version": "1.13.0" "version": "2.0.0"
} }

7
packages/claude/dist/rag/scripts/client.py vendored
View File

@ -16,7 +16,7 @@ import urllib.parse
import urllib.request import urllib.request
MAX_RESPONSE_BYTES = 2 * 1024 * 1024 MAX_RESPONSE_BYTES = 2 * 1024 * 1024
ENGINES = ("rag", "graphrag", "crzp") ENGINES = ("rag", "graphrag") # CRZP is part of both answers
STATUSES = {"ok", "no_answer", "ambiguous", "error"} STATUSES = {"ok", "no_answer", "ambiguous", "error"}
@ -103,7 +103,7 @@ def error_result(reason):
"forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.", "forbidden": "Pre tento nástroj nemáš potrebné oprávnenie.",
"invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.", "invalid_query": "Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.",
"question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.", "question_mismatch": "Odpoveď patrí k inej otázke a nezobrazí sa. Zopakuj aktuálnu otázku.",
"engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag, /graphrag alebo /crzp.", "engine_mismatch": "Príkaz a zvolený nástroj sa nezhodujú. Použi nástroj podľa /rag alebo /graphrag.",
"crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.", "crzp_unavailable": "CRZP momentálne neodpovedá alebo obmedzil počet požiadaviek. Skús to o chvíľu.",
"model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.", "model_unavailable": "Jazykový model momentálne nie je dostupný. Skús otázku neskôr.",
'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.', 'verification_unavailable':'Finálne overenie odpovede momentálne nie je dostupné. Skús otázku neskôr; nejde o potvrdenie chýbajúcich podkladov.',
@ -217,14 +217,13 @@ def query_backend(query: str, engine: str, *, mode="answer", limit=5, config=Non
if ( if (
engine not in ENGINES engine not in ENGINES
or mode not in {"answer", "retrieve"} or mode not in {"answer", "retrieve"}
or engine == "crzp" and mode != "answer"
or type(limit) is not int or type(limit) is not int
or not 1 <= limit <= 20 or not 1 <= limit <= 20
or not isinstance(query, str) or not isinstance(query, str)
): ):
raise ClientError("invalid_query") raise ClientError("invalid_query")
query = query.strip() query = query.strip()
command = re.match(r"^/(rag|graphrag|crzp)(?=\s|$)", query, re.I) command = re.match(r"^/(rag|graphrag)(?=\s|$)", query, re.I)
if command: if command:
if command[1].lower() != engine: if command[1].lower() != engine:
raise ClientError("engine_mismatch") raise ClientError("engine_mismatch")

BIN
packages/claude/dist/zpwiki-crzp.zip vendored
View File

Binary file not shown.

BIN
packages/claude/dist/zpwiki-graphrag.zip vendored
View File

Binary file not shown.

BIN
packages/claude/dist/zpwiki-rag.zip vendored
View File

Binary file not shown.

28
packages/claude/mcp_server.py
View File

@ -23,7 +23,7 @@ QUESTION_POLICY = (
"Load each tool definition with the host tool search when tools are deferred; use the exact loaded " "Load each tool definition with the host tool search when tools are deferred; use the exact loaded "
"identifier and schema. Put the question into the query argument, never an argument named question. " "identifier and schema. Put the question into the query argument, never an argument named question. "
"A tool-not-loaded or invalid-argument error is a host invocation error, not a DNS failure. " "A tool-not-loaded or invalid-argument error is a host invocation error, not a DNS failure. "
"Send the user's latest question verbatim, removing only a leading /rag, /graphrag or /crzp command. Do not " "Send the user's latest question verbatim, removing only a leading /rag or /graphrag command. Do not "
"paraphrase, translate, correct names or add/drop conditions; the backend resolves misspelled and partial names. " "paraphrase, translate, correct names or add/drop conditions; the backend resolves misspelled and partial names. "
"One call per question; several explicit questions in one message are separate calls in order. " "One call per question; several explicit questions in one message are separate calls in order. "
"If the previous result contained conversation_context.token and the new message continues it (a name chosen " "If the previous result contained conversation_context.token and the new message continues it (a name chosen "
@ -35,6 +35,9 @@ QUESTION_POLICY = (
ANSWER_PRESENTATION_POLICY = ( ANSWER_PRESENTATION_POLICY = (
"When accepted_for_delivery is true, present answer exactly as returned, with its citations, notices and " "When accepted_for_delivery is true, present answer exactly as returned, with its citations, notices and "
"recognized-name lines; add no facts, guesses or commentary about checks, tokens or diagnostics. " "recognized-name lines; add no facts, guesses or commentary about checks, tokens or diagnostics. "
"The whole reply is that answer text unchanged: nothing before or after it (no introduction, summary, "
"rephrasing, evaluation, advice, question back or offer of further help); for anything the answer lacks, "
"call the tool again instead of adding it. "
"For status ambiguous show the clarification and wait; for no_answer show only the supplied message and do not " "For status ambiguous show the clarification and wait; for no_answer show only the supplied message and do not "
"claim nonexistence; for error say a verified answer is unavailable and never write a replacement. After a " "claim nonexistence; for error say a verified answer is unavailable and never write a replacement. After a "
"timeout the user may repeat the same question; the backend then returns the finished answer. " "timeout the user may repeat the same question; the backend then returns the finished answer. "
@ -81,20 +84,6 @@ async def zpwiki_graphrag(query: str, limit: int = 5, context_token: str | None
) )
async def zpwiki_crzp(query: str, limit: int = 5, context_token: str | None = None) -> CallToolResult:
"""Verified answer from CRZP (Central Register of Theses, all Slovak schools): works by title, student,
supervisor, opponent, school, year, type or topic; what a work is about, its keywords (tags), counts,
schools and related works. Metadata and abstracts only.
Pass the full question in query, not question. Optional limit is an integer.
For a follow-up ("a v roku 2023?", "z akých škôl boli tí študenti?", "o čom je tá druhá?") copy
conversation_context.token from the last zpwiki_crzp result into context_token.
Present the accepted answer with its CRZP links; no_answer means no matching CRZP record was found.
"""
return tool_result(answer_for_presentation(await asyncio.to_thread(query_backend, query, "crzp", limit=limit,
**({'context_token':context_token} if context_token is not None else {}))))
async def zpwiki_health() -> CallToolResult: async def zpwiki_health() -> CallToolResult:
"""Read readiness and data versions of both tools without calling a language model.""" """Read readiness and data versions of both tools without calling a language model."""
return tool_result(await asyncio.to_thread(backend_health)) return tool_result(await asyncio.to_thread(backend_health))
@ -104,15 +93,14 @@ def create_server(**kwargs):
server = FastMCP( server = FastMCP(
"ZP Wiki", "ZP Wiki",
instructions=( instructions=(
"Use the tool selected by the user's /rag, /graphrag or /crzp command. Without a command use zpwiki_rag; " "Use the tool selected by the user's /rag or /graphrag command. Without a command use zpwiki_rag; "
"zpwiki_graphrag serves graph relationships, related topics and metadata counts; zpwiki_crzp searches " "zpwiki_graphrag serves graph relationships, related topics and metadata counts. Both also answer from "
"the national CRZP register of theses of all Slovak schools (use it for /crzp or when the user asks " "the national CRZP register of theses of all Slovak schools; CRZP parts are marked \"V CRZP\". "
"about CRZP). "
"Tools return backend-verified answers. " + QUESTION_POLICY + ANSWER_PRESENTATION_POLICY "Tools return backend-verified answers. " + QUESTION_POLICY + ANSWER_PRESENTATION_POLICY
), ),
**kwargs, **kwargs,
) )
for handler in (zpwiki_rag, zpwiki_graphrag, zpwiki_crzp, zpwiki_health): for handler in (zpwiki_rag, zpwiki_graphrag, zpwiki_health):
server.add_tool( server.add_tool(
handler, handler,
annotations=ToolAnnotations( annotations=ToolAnnotations(

2
test/test_academic.py
View File

@ -74,7 +74,7 @@ def test_latest_meeting_is_selected_by_date_even_when_profile_order_is_reversed(
"Kedy mala Alena Nová posledné stretnutie a o čom bolo?", data, engine=engine "Kedy mala Alena Nová posledné stretnutie a o čom bolo?", data, engine=engine
) )
assert r["status"] == "ok" and "2026-05-05" in r["answer"] assert r["status"] == "ok" and "2026-05-05" in r["answer"]
assert "Implementovaný" in r["answer"] and "Plán:" in r["answer"] assert "Implementovaný" in r["answer"] and "Úloha zo stretnutia:" in r["answer"]
assert "prvý návrh" not in r["answer"] assert "prvý návrh" not in r["answer"]
assert all(e.get("date") == "2026-05-05" for e in r["evidence"]) assert all(e.get("date") == "2026-05-05" for e in r["evidence"])

2
test/test_answer_explanations.py
View File

@ -60,7 +60,7 @@ def test_meeting_prose_retains_planned_vs_executed_without_copying_imperatives()
'text':'Má vyhodnotiť systém.' if r['phase']=='planned' else 'Funguje agent RAG.'} for r in payload['evidence']]} 'text':'Má vyhodnotiť systém.' if r['phase']=='planned' else 'Funguje agent RAG.'} for r in payload['evidence']]}
return {'choices':[{'message':{'content':json.dumps(value)}}]} return {'choices':[{'message':{'content':json.dumps(value)}}]}
out=s.synthesize(result,plan,complete=model,diagnostics=True) out=s.synthesize(result,plan,complete=model,diagnostics=True)
assert out['synthesis']['status']=='accepted' and 'Ďalší postup: Má vyhodnotiť systém.' in out['answer'] assert out['synthesis']['status']=='accepted' and 'Úlohy: Má vyhodnotiť systém.' in out['answer']
assert 'Vyhodnoťte' not in out['answer'] and 'Vyhodnotil' not in out['answer'] assert 'Vyhodnoťte' not in out['answer'] and 'Vyhodnotil' not in out['answer']

29
test/test_claude_packages.py
View File

@ -410,8 +410,8 @@ def test_config_file_permissions_and_environment_override(tmp_path, monkeypatch)
@pytest.mark.parametrize("engine", ["rag", "graphrag"]) @pytest.mark.parametrize("engine", ["rag", "graphrag"])
def test_zip_is_reproducible_and_runs_without_repository(tmp_path, backend, engine): def test_zip_is_reproducible_and_runs_without_repository(tmp_path, backend, engine):
one = build_package(engine, tmp_path / "one") one = build_package(engine, tmp_path / "one", tmp_path / "one")
two = build_package(engine, tmp_path / "two") two = build_package(engine, tmp_path / "two", tmp_path / "two")
assert one["sha256"] == two["sha256"] assert one["sha256"] == two["sha256"]
archive_path = tmp_path / "one" / one["archive"] archive_path = tmp_path / "one" / one["archive"]
folder = tmp_path / "installed" folder = tmp_path / "installed"
@ -456,18 +456,20 @@ def test_zip_is_reproducible_and_runs_without_repository(tmp_path, backend, engi
assert raw["complete"] is False and raw["truncated"] is True assert raw["complete"] is False and raw["truncated"] is True
def test_committed_archives_match_current_sources(tmp_path): def test_committed_skill_folders_match_current_sources(tmp_path):
manifest = json.loads((ROOT / "packages/claude/dist/packages.json").read_text()) dist = ROOT / "packages/claude/dist"
manifest = json.loads((dist / "packages.json").read_text())
assert not list(dist.glob("*.zip")) # archives are made on request only (--zip)
for row in manifest["packages"]: for row in manifest["packages"]:
engine = row["name"] engine = row["name"]
rebuilt = build_package(engine, tmp_path) rebuilt = build_package(engine, tmp_path)
actual = ROOT / "packages/claude/dist" / row["archive"] assert rebuilt["manifest_sha256"] == row["manifest_sha256"]
assert hashlib.sha256(actual.read_bytes()).hexdigest() == rebuilt["sha256"] == row["sha256"] built = sorted(p.relative_to(tmp_path / engine) for p in (tmp_path / engine).rglob("*") if p.is_file())
# The unpacked folder in dist/ holds exactly the archive's files. kept = sorted(p.relative_to(dist / engine) for p in (dist / engine).rglob("*")
import zipfile if p.is_file() and "__pycache__" not in p.parts)
with zipfile.ZipFile(actual) as archive: assert built == kept
for member in archive.namelist(): for path in built:
assert (ROOT / "packages/claude/dist" / member).read_bytes() == archive.read(member), member assert (dist / engine / path).read_bytes() == (tmp_path / engine / path).read_bytes(), path
def test_mcp_stdio_initialization_discovery_call_and_rejection(backend): def test_mcp_stdio_initialization_discovery_call_and_rejection(backend):
@ -497,7 +499,6 @@ def test_mcp_stdio_initialization_discovery_call_and_rejection(backend):
assert {row.name for row in listed.tools} == { assert {row.name for row in listed.tools} == {
"zpwiki_rag", "zpwiki_rag",
"zpwiki_graphrag", "zpwiki_graphrag",
"zpwiki_crzp",
"zpwiki_health", "zpwiki_health",
} }
for tool in listed.tools: for tool in listed.tools:
@ -634,7 +635,7 @@ def test_remote_mcp_authentication_and_full_protocol(backend, monkeypatch):
) as (read, write, _): ) as (read, write, _):
async with ClientSession(read, write) as session: async with ClientSession(read, write) as session:
await session.initialize() await session.initialize()
assert len((await session.list_tools()).tools) == 4 assert len((await session.list_tools()).tools) == 3
for engine in ["rag", "graphrag"]: for engine in ["rag", "graphrag"]:
result = await session.call_tool( result = await session.call_tool(
"zpwiki_" + engine, {"query": "Práca Evy Novej?"} "zpwiki_" + engine, {"query": "Práca Evy Novej?"}
@ -817,7 +818,7 @@ def test_oauth_discovery_and_full_flow_with_mcp(oauth_app):
) as (read, write, _): ) as (read, write, _):
async with ClientSession(read, write) as session: async with ClientSession(read, write) as session:
await session.initialize() await session.initialize()
assert len((await session.list_tools()).tools) == 4 assert len((await session.list_tools()).tools) == 3
health = await session.call_tool("zpwiki_health") health = await session.call_tool("zpwiki_health")
assert not health.isError and health.structuredContent["ready"] assert not health.isError and health.structuredContent["ready"]
result = await session.call_tool( result = await session.call_tool(

115
test/test_combined_answers.py Normal file
View File

@ -0,0 +1,115 @@
"""ZP Wiki + CRZP in one answer: order, labels, clarification, follow-ups, CRZP timeout."""
import time
import pytest
from tools.answers import combined as C
WIKI = {"status": "ok", "reason": None, "accepted_for_delivery": True, "claims": [], "index_version": {"version": "v"},
"complete": True, "truncated": False, "truncation_reasons": [],
"answer": "**Ján Pták — Diplomová práca 2027**\n\nVedúci práce: Daniel Hladek.\n\nZdroje: https://zp.test/jan",
"source_urls": ["https://zp.test/jan"],
"delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": "x"}}
@pytest.fixture
def combo(monkeypatch):
monkeypatch.setenv("SEARCH_API_KEY", "combined-test-key")
monkeypatch.setenv("ANSWER_WITH_CRZP", "1")
state = {"part": None, "match": {"name": "Ján Pták", "student": True}}
monkeypatch.setattr("tools.crzp.answering.wiki_bridge.wiki_match", lambda q, db_file=None: state["match"])
monkeypatch.setattr(C, "crzp_part", lambda q, person, complete=None, student=True: state["part"])
state["deliver"] = lambda q, **kw: dict(WIKI)
return state
def ask(combo, question, token=None):
return C.answer(question, engine="rag", context_token=token, deliver=lambda q, **kw: combo["deliver"](q, **kw))
def test_wiki_first_then_the_marked_crzp_part(combo):
combo["part"] = {"mode": "person", "text": "- bakalárska práca 2025 „IoT“ — vedúci Pavol Galajda; oponent Daniel Hládek",
"rows": [], "urls": ["https://opac.test/1"], "people": [], "works": []}
result = ask(combo, "Kto vedie Jána Ptáka?")
text = result["answer"]
assert text.index("Vedúci práce: Daniel Hladek") < text.index("**V CRZP**") < text.index("Pavol Galajda")
assert result["source_urls"] == ["https://zp.test/jan", "https://opac.test/1"] and result["status"] == "ok"
import packages.claude.client as client
client.validate_metadata(result)
assert client.validated_delivery_check(result, "Kto vedie Jána Ptáka?", "rag")
def test_without_crzp_the_wiki_answer_is_unchanged_but_for_wording(combo):
result = ask(combo, "Kedy mal Ján Pták posledné stretnutie?")
assert "V CRZP" not in result["answer"] and result["answer"].startswith("**Ján Pták")
def test_a_work_only_crzp_has_is_asked_for_and_the_choice_continues(combo, monkeypatch):
combo["part"] = {"mode": "person", "text": "x", "rows": [], "urls": [], "people": [],
"works": [{"source": "crzp", "label": "bakalárska práca 2025 „IoT“", "type": "bachelor_thesis", "key": "k"}]}
monkeypatch.setattr(C, "wiki_works", lambda person: {"Diplomová práca 2027": {"type": "diploma_thesis", "topic": "Agent"}})
asked = ask(combo, "Aké technológie využíva Ján Pták vo svojej práci?")
assert asked["status"] == "ambiguous" and "1. Diplomová práca 2027 „Agent“ (ZP Wiki)" in asked["answer"]
assert "2. bakalárska práca 2025 „IoT“ (CRZP)" in asked["answer"]
seen = []
combo["deliver"] = lambda q, **kw: (seen.append(q), dict(WIKI))[1]
ask(combo, "1", asked["conversation_context"]["token"])
assert seen == ["Aké technológie využíva Ján Pták vo svojej diplomovej práci?"]
chosen = []
monkeypatch.setattr(C, "_answer_choice", lambda q, c, **kw: chosen.append(c) or dict(WIKI))
ask(combo, "tú bakalársku", asked["conversation_context"]["token"])
assert chosen[0]["source"] == "crzp"
def test_schools_of_the_people_of_the_previous_answer(combo):
combo["part"] = {"mode": "similar", "text": "- Maksym Zatirka — Technická univerzita v Košiciach: „X“", "rows": [],
"urls": [], "people": [("Eva Malá", "Slovenská technická univerzita v Bratislave")]}
first = ask(combo, "Kto pracoval na podobnej téme ako Ján Pták?")
combo["match"] = None
schools = ask(combo, "a z akých škôl boli tí študenti?", first["conversation_context"]["token"])
assert "- Ján Pták — Technická univerzita v Košiciach (KEMT FEI)" in schools["answer"]
assert "- Eva Malá — Slovenská technická univerzita v Bratislave" in schools["answer"]
def test_a_slow_crzp_does_not_hold_the_wiki_answer(combo, monkeypatch):
monkeypatch.setattr(C, "CRZP_SECONDS", 0.5)
monkeypatch.setattr(C, "crzp_part", lambda *a, **k: (time.sleep(4), None)[1])
started = time.monotonic()
result = ask(combo, "Kto vedie Jána Ptáka?")
assert result["status"] == "ok" and time.monotonic() - started < 2.0
def test_a_forged_combined_token_is_refused(combo):
first = ask(combo, "Kto vedie Jána Ptáka?")
token = first["conversation_context"]["token"]
assert ask(combo, "a z akých škôl?", token[:-3] + "abc")["reason"] == "invalid_context"
def test_crzp_people_and_keywords_link_to_the_wiki_graph_by_code():
from tools.crzp.storage.graph import link_pairs
rows = [{"school": "Technická univerzita v Košiciach", "authors": ["Pták, Ján"], "supervisors": ["Galajda, Pavol"],
"opponents": [], "keywords": ["Speech Recognition", "iot"]},
{"school": "Iná univerzita", "authors": ["Pták, Jan"], "supervisors": [], "opponents": [], "keywords": []}]
people, keywords = link_pairs(rows, ["Ján Pták", "Daniel Hladek"], ["rozpoznávanie reči", "Dialógový systém"])
assert people == [{"crzp": "Pták, Ján", "wiki": "Ján Pták"}] # only at TUKE, the namesake elsewhere is not linked
assert keywords == [{"key": "speech recognition", "wiki": "rozpoznávanie reči"}] # through the reviewed synonyms
def test_the_wiki_question_which_person_wins_over_crzp(combo):
combo["match"] = None
combo["part"] = {"mode": "explicit", "status": "ambiguous", "text": "Menu „Ján“ zodpovedá v CRZP…", "rows": [],
"urls": [], "people": []}
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "ambiguous_person",
"answer": "Ktorého Jána myslíš? Ján Holp, Ján Pták."}
result = ask(combo, "Na čom pracoval Ján?")
assert result["answer"] == "Ktorého Jána myslíš? Ján Holp, Ján Pták."
combo["deliver"] = lambda q, **kw: {**WIKI, "status": "ambiguous", "reason": "missing_subject",
"answer": "Uveď meno študenta."}
combo["part"] = {**combo["part"], "status": "ok", "text": "Školitelia prác …"}
assert "Školitelia prác" in ask(combo, "Kto viedol najviac prác o strojovom učení?")["answer"]
def test_an_unknown_person_goes_to_crzp_only_for_a_thesis_question():
assert C.THESIS.search(C.normalized("O čom je bakalárska práca Jána Nováka?"))
assert not C.THESIS.search(C.normalized("Na čom pracuje Peter Novák?"))
assert not C.ABOUT.search(C.normalized("Čomu sa venuje Jakub Schwarc?"))

73
test/test_content_selection_ids.py
View File

@ -280,3 +280,76 @@ def test_graph_narrative_request_uses_shared_plan_and_content_evidence(monkeypat
assert result['status']=='ok' and result['accepted_for_delivery'] assert result['status']=='ok' and result['accepted_for_delivery']
assert all(name in result['answer'] for name in names) assert all(name in result['answer'] for name in names)
assert 'problém s modelom' in result['answer'] and 'experiment' in result['answer'] assert 'problém s modelom' in result['answer'] and 'experiment' in result['answer']
def test_a_status_record_naming_tools_answers_which_technologies_were_used():
from tools.answers.content_selection import facets_for, named_tools
q = "Aké technológie použil Ján Pták vo svojej diplomovej práci?"
assert "methods" in facets_for("- Sú Skills aj MCP server. RAG vie v ZPwiki vyhľadávať.", "progress", q)
assert "methods" in facets_for("- začíname pracovať na GraphRAG a Neo4J", "progress", q)
assert "methods" in facets_for("- Funguje vektorové a textové vyhľadávanie.", "result", q)
assert "methods" not in facets_for("- Vytvorte nasadenie agenta pomocou DockerCompose.", "plan", q)
assert {"MCP", "RAG", "GraphRAG", "Neo4J", "Fast120", "me5mini"} <= set(
named_tools("MCP, RAG, GraphRAG, Neo4J, Fast120 a me5mini")) and not named_tools("Stretnutie 5.11.2026")
def test_a_meeting_question_is_not_narrowed_to_a_guessed_aspect():
from tools.answers.question_catalog import Route
from tools.answers.question_plan import direct_spec
from tools.retrieval.person_names import resolve_person_names
catalog = [{"name": "Jakub Schwarc", "kind": "Person", "source_document": "pages/students/2022/jakub_schwarc/README.md"}]
for question, facets in [("Kedy mal Jakub Schwarc posledné stretnutie a čo na ňom riešil?", []),
("Aké výsledky mal Jakub Schwarc na poslednom stretnutí?", ["results"])]:
route = Route(type="meeting", mode="meeting", facets=["results"], meeting="latest")
spec = direct_spec(question, resolve_person_names(question, catalog), route=route)
assert spec.facets == facets, question
def test_an_aspect_named_in_the_question_wins_over_the_classifier():
from tools.answers.question_catalog import Route
from tools.answers.question_plan import direct_spec
from tools.retrieval.person_names import resolve_person_names
catalog = [{"name": "Jakub Schwarc", "kind": "Person", "source_document": "pages/students/2022/jakub_schwarc/README.md"}]
question = "Aké úlohy dostal Jakub Schwarc?"
route = Route(type="work_details", mode="facets", facets=["assignment"])
assert direct_spec(question, resolve_person_names(question, catalog), route=route).facets == ["plans"]
def test_final_pass_writes_full_sources_and_drops_system_notices():
from tools.answers.finalize import finalize
text = ("**Ján Pták — Diplomová práca 2027**\n\nNázov / téma zo zdroja: Agent\n\nPoužíva RAG. [1]\n\n"
"Vedúci je určený podľa autora profilovej stránky študenta v ZP Wiki.\n\n"
"Profil obsahuje aj záznamy bez úplného dátumu; nemožno spoľahlivo určiť, či sú novšie. Rok nedopĺňam z roku práce.\n\n"
"Zdroje: [1](https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak)\n\n[1]: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak")
out = finalize(text)
assert "[1]" not in out and out.endswith("Zdroje: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak")
assert "Téma: Agent" in out and "Vedúci je určený" not in out and "Profil obsahuje" not in out
only_missing = "Zadanie a ciele:\n\nJakub Schwarc — Diplomová práca 2027: zadanie a ciele: v podkladoch chýbajú doložené údaje."
assert finalize(only_missing) == "ZP Wiki k tomu nemá záznam."
def test_final_pass_simplifies_technical_bullets_and_collapses_assignment_only_students():
from tools.answers.finalize import finalize
text = ("**Samuel Džurina**\n\n- Bakalárska práca 2026; priebežný stav; záznam 2025-05-13: Chatbot funguje.\n\n"
"**Martin Jancura**\n\nMartin Jancura — téma je v zadaní, téme alebo pláne práce; zaznamenaný výsledok k nej sa nenašiel.\n\n"
"- Diplomový projekt; podklad: ## Diplomový projekt 2\n\n"
"**Dominik Nagy**\n\n- Diplomová práca 2022; podklad: *Meno vedúceho*: Ing. Daniel Hládek, PhD.\n\n"
"- Diplomová práca 2022; téma alebo názov: *Názov diplomovej práce*: Neurónový strojový preklad\n\n"
"Zdroje: https://a.test/x, https://a.test/y")
out = finalize(text)
assert "- Bakalárska práca 2026 (2025-05-13): Chatbot funguje." in out
assert "- Diplomová práca 2022: Neurónový strojový preklad" in out and "Hládek" not in out and "##" not in out
assert out.count("Martin Jancura") == 1 and "Téma je len v zadaní alebo pláne, bez zaznamenaného výsledku: Martin Jancura." in out
def test_related_topics_find_inflected_phrases_and_synonyms():
from tools.retrieval.synonyms import base_name, variants_pattern
from tools.answers.context_policy import normalized
assert base_name("rozpoznávaním reči") == base_name("speech recognition") == "rozpoznávanie reči"
assert variants_pattern("speech recognition").search(normalized("Funguje automatické rozpoznávanie reči."))
assert base_name("Python") == "Python"
def test_vyuziva_asks_for_methods_like_pouziva():
from tools.answers.question_intent import requested_facets
assert requested_facets("Aké technológie využíva Ján Pták vo svojej diplomovej práci?") == ["methods"]

13
test/test_conversation_context.py
View File

@ -244,3 +244,16 @@ def test_client_transports_context_without_rewriting_query(monkeypatch):
result=client.query_backend('Daj obe.','rag',config=client.Config('http://localhost:8000','test'),context_token='opaque') result=client.query_backend('Daj obe.','rag',config=client.Config('http://localhost:8000','test'),context_token='opaque')
assert captured['query']=='Daj obe.' and captured['context_token']=='opaque' assert captured['query']=='Daj obe.' and captured['context_token']=='opaque'
assert client.answer_for_presentation(result)['conversation_context']['token']=='opaque' assert client.answer_for_presentation(result)['conversation_context']['token']=='opaque'
@pytest.mark.parametrize('q',['Kto bol jeho vedúci?','Aké technológie použil?','Kedy mal posledné stretnutie?',
'Aké výsledky dosiahol tento študent?'])
def test_pronoun_or_aspect_only_question_continues_with_the_previous_person(data,q):
r=resolve_question(q,token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION)
assert 'Ján Holp' in r.query and r.clarification is None
@pytest.mark.parametrize('q',['Kto robil chatbot?','Kto pracoval s Cassandrou?'])
def test_a_new_topic_is_not_given_the_previous_person(data,q):
r=resolve_question(q,token=work_context(data)['token'],catalog=CATALOG,index=data,version=VERSION)
assert 'Ján Holp' not in r.query

600
tools/answers/combined.py Normal file
View File

@ -0,0 +1,600 @@
"""ZP Wiki and CRZP in one answer for /rag and /graphrag.
Every question is answered from ZP Wiki (the normal RAG / GraphRAG path) and,
at the same time, from CRZP when CRZP can add something (see crzp_plan):
- a person ZP Wiki knows: that person's works in CRZP at Technická univerzita
v Košiciach (supervisor, opponent, abstract, keywords the wiki may lack);
- a topic or "similar works": CRZP works on the same topic, with their schools;
- a question naming CRZP, another school or opponents: CRZP is the main source.
Meetings, tasks, progress and results stay ZP Wiki only. The wiki part comes
first, the CRZP part is marked with its source; nothing is mixed into one fact.
When a person has different works in the two sources (a diploma thesis in the
wiki, a bachelor thesis in CRZP) and the question does not say which one, the
answer asks; the reply ("2", "bakalársku") continues through a signed token
that also carries the wiki's own token and the people of the last answer
(for "a z akých škôl boli tí študenti?").
"""
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import os
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout
from functools import lru_cache
from tools.answers.context_policy import normalized
from tools.answers.delivery_contract import fingerprint
CRZP_SECONDS = float(os.getenv("CRZP_COMBINED_SECONDS", "45"))
LIFETIME = 24 * 3600
PREFIX = "zc1."
TUKE = "Technická univerzita v Košiciach"
WIKI_SCHOOL = "Technická univerzita v Košiciach (KEMT FEI)"
# What only the wiki records: CRZP is not asked.
WIKI_ONLY = re.compile(r"\b(?:stretnut\w*|pokrok\w*|progres\w*|uloh\w*|zadan\w*|plan\w*|termin\w*|tyzd\w*|"
r"repozitar\w*|git\w*|stav\w*|poznam\w*|konzult\w*|zaciat\w*|studi\w*|obhaj\w*|ukonc\w*|"
r"dokonc\w*|uspes\w*|statnic\w*|hodnot\w*|znamk\w*|problem\w*|vtedy|posledn\w*)\b")
EXPLICIT = re.compile(r"\b(?:crzp|registr\w*|register|celoslovensk\w*|na slovensku|inych skol|inej skole|"
r"vsetkych skol|ostatnych skol|oponent\w*|oponoval\w*)\b")
METHODS = re.compile(r"\b(?:technolog\w*|nastroj\w*|metod\w*|pouziv\w*|pouzil\w*|vyuziv\w*|vyuzil\w*|framework\w*)\b")
ABOUT = re.compile(r"\b(?:o com|zhrn\w*|abstrakt\w*|ciel\w*)\b") # not "čomu sa venuje": the current work
# A question about a thesis (not "na čom pracuje", which CRZP's submitted works cannot answer).
THESIS = re.compile(r"\b(?:prac(?:a|e|u|i|ou|ach|ami)?|bakalar\w*|diplom\w*|dizerta\w*|habilita\w*|tem(?:a|u|y|ou)|"
r"oponent\w*|vedu\w*|skolitel\w*|abstrakt\w*|klucov\w*|tagy?)\b")
WIKI_CLARIFICATION_YIELDS = {"missing_subject", "unsupported_content_question"}
REFUSED = {"request_out_of_scope", "irrelevant", "unsupported_request", "out_of_scope"}
RELATED = re.compile(r"\b(?:suvis\w*|pribuzn\w*)\b")
SIMILAR = re.compile(r"\b(?:podobn\w*|suvisiac\w*|pribuzn\w*|spolupra\w*|rovnak\w* tem\w*)\b")
TOPIC_LIST = re.compile(r"^\W*(?:kto|ktor\w*|ak\w*)\b")
SCHOOLS = re.compile(r"\b(?:z|zo|na) (?:akych|akej|ktorych|ktorej) (?:skol|univerzit|fakult)\w*|\bkde (?:studoval\w*|studuj\w*)"
r"|\b(?:ktor|ak)\w* (?:skol|univerzit)\w*")
CHOICE_WIKI = re.compile(r"\b(?:wiki|diplom\w*|dp)\b")
CHOICE_CRZP = re.compile(r"\b(?:crzp|bakalar\w*|bp)\b")
_shared = {"pool": None}
_shared_lock = threading.Lock()
def _pool() -> ThreadPoolExecutor:
"""One pool for all questions (no new threads per request)."""
with _shared_lock:
if _shared["pool"] is None:
_shared["pool"] = ThreadPoolExecutor(max_workers=int(os.getenv("ANSWER_COMBINED_WORKERS", "6")),
thread_name_prefix="answer")
return _shared["pool"]
def enabled() -> bool:
return os.getenv("ANSWER_WITH_CRZP", "1") != "0"
# ------------------------------------------------------------------ token
def _key() -> bytes | None:
secret = os.getenv("SEARCH_API_KEY", "")
return hashlib.sha256(("zp-combined-v1\0" + secret).encode()).digest() if secret else None
def _b64(value: bytes) -> str:
return base64.urlsafe_b64encode(value).decode().rstrip("=")
def seal(payload: dict) -> str | None:
key = _key()
if key is None:
return None
body = _b64(json.dumps({**payload, "expires": int(time.time()) + LIFETIME}, ensure_ascii=False,
separators=(",", ":")).encode())
token = PREFIX + body + "." + _b64(hmac.digest(key, body.encode(), "sha256"))
return token if len(token) <= 16000 else None
def unseal(token: str | None) -> dict | None:
"""The combined context, or None for a token that is not ours (a plain wiki token)."""
if not isinstance(token, str) or not token.startswith(PREFIX) or _key() is None:
return None
try:
body, signature = token[len(PREFIX):].split(".")
expected = hmac.digest(_key(), body.encode(), "sha256")
if not hmac.compare_digest(base64.b64decode(signature + "=" * (-len(signature) % 4), altchars=b"-_"), expected):
raise ValueError
value = json.loads(base64.b64decode(body + "=" * (-len(body) % 4), altchars=b"-_"))
if value.get("expires", 0) < time.time():
return {"error": "expired_context"}
return value
except (ValueError, TypeError, UnicodeError):
return {"error": "invalid_context"}
# ------------------------------------------------------------------ wiki facts
_index_cache: dict = {}
_index_lock = threading.Lock()
def _index():
"""The published knowledge index, reloaded only when its version changes."""
from tools.knowledge.store import load_index
from tools.rag.common import DB_FILE
from tools.sync.index_state import read_version
version = read_version(DB_FILE.parent)
key = (version or {}).get("version")
with _index_lock:
if _index_cache.get("key") != key:
_index_cache.update(key=key, index=load_index(DB_FILE.parent, version))
return _index_cache["index"]
def wiki_units(person: str) -> list[dict]:
try:
return [u for u in (_index() or {}).get("units", []) if u.get("person") == person]
except Exception:
return []
def wiki_works(person: str) -> dict[str, dict]:
"""work label -> {"type", "topic"} of the person's ZP Wiki works."""
works = {}
for unit in wiki_units(person):
label = unit.get("work_label")
if not label:
continue
work = works.setdefault(label, {"type": unit.get("work_type"), "topic": None})
if unit.get("kind") == "topic" and not work["topic"] and not unit["text"].strip().endswith(":"):
work["topic"] = re.sub(r"^\s*(?:Téma|Názov)\s*:\s*", "", unit["text"]).strip()
return works
FRAMEWORKS = {"python", "fastapi", "flask", "django", "streamlit", "react", "docker", "kubernetes", "git", "github",
"gitlab", "pandas", "numpy", "jupyter", "colab", "linux", "javascript", "typescript", "java", "html", "css",
"llm", "ai", "nlp", "veľký jazykový model", "umelá inteligencia", "zpwiki", "openwebui", "skills", "skill"}
def wiki_terms(person: str) -> list[str]:
"""The person's most specific topics and tools (reviewed topic groups first, frameworks never):
the words to look for in CRZP to find similar works."""
from tools.answers.content_selection import named_tools
from tools.retrieval.synonyms import base_name, group_of
counts: dict[str, float] = {}
for unit in wiki_units(person):
if unit.get("phase") not in {"result", "progress", "assignment"} and unit.get("kind") != "topic":
continue
for concept in unit.get("concepts") or []:
if not isinstance(concept, str):
continue
if len(group_of(concept)) > 1:
name, weight = base_name(concept), 2.0
elif named_tools(concept) and len(concept.split()) <= 2 and not re.search(r"[./]", concept):
name, weight = concept, 1.0
else:
continue
if normalized(name) not in FRAMEWORKS:
counts[name] = counts.get(name, 0) + weight
return sorted(counts, key=lambda n: (-counts[n], n))[:4]
def wiki_profile(person: str) -> str:
"""Topic titles and terms of the person's works: the text similar CRZP works are ranked against."""
works = wiki_works(person)
return ". ".join([*(w["topic"] for w in works.values() if w.get("topic")), ", ".join(wiki_terms(person))])
# ------------------------------------------------------------------ CRZP part
def crzp_plan(question: str, person: str | None, complete=None, student: bool = True):
"""(mode, CrzpQuery) or None when CRZP adds nothing to this question."""
from tools.crzp.answering.understand import Person, understand
low = normalized(question)
explicit = bool(EXPLICIT.search(low))
if WIKI_ONLY.search(low) and not explicit:
return None
query = understand(question, complete=complete)
if query.intent == "refused":
return None
other_school = query.school not in (None, "tuke") or explicit
if person and not other_school:
if SIMILAR.search(low) and student:
return "similar", query
role = query.people[0].role if query.people else "any"
if student:
role = "author" if role == "author" else "any" # "Kto vedie Jána Ptáka?": his own works
elif role not in ("supervisor", "opponent"):
role = "supervisor" # a supervisor's works are those he or she led
query.people = [Person(name=person, role=role, typed=person)]
query.school, query.unit, query.from_model, query.title = "tuke", None, True, None
query.topics, query.alternatives = [], []
return "person", query
if query.people and not person and query.search_text() and THESIS.search(low):
return "explicit", query # a person ZP Wiki does not know: CRZP knows the theses of all schools
if query.topics and not query.people and RELATED.search(low):
query.intent = "keywords" # "Ktoré témy súvisia s X?": the tags of CRZP works about X
return "explicit", query
if query.topics and not query.people and query.intent in ("relation", "count", "keywords", "schools"):
return "explicit", query # "Kto viedol najviac prác o X?", "Koľko prác o X…": computed over CRZP
if (other_school or ((query.school or query.unit) and not query.people)) and query.search_text():
return "explicit", query # another school, CRZP itself, opponents, or a school/department as a whole
if query.topics and not query.people and (TOPIC_LIST.search(low) or SIMILAR.search(low)):
return "topic", query
return None
def clean_crzp(text: str) -> str:
"""CRZP answer text inside a combined answer: its own source lines go to the common "Zdroje CRZP" line."""
lines = [l for l in text.split("\n") if not re.match(r"^\s*(?:-\s*)?Zdroj:\s", l)]
return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()
def explicit_part(question: str, result: dict) -> dict | None:
"""A whole CRZP answer (CRZP is the main source), with its conversation token for follow-ups."""
from tools.crzp.answering import answer as crzp, context as crzp_context
if result.get("status") not in ("ok", "ambiguous"):
return None
rows = result.pop("_shown", None) or []
asked = result.pop("_asked", None)
options = result.pop("_options", None)
token = None
if asked is not None:
issued = crzp_context.issue(question, asked, rows, options)
token = issued["token"] if issued else None
return {"mode": "explicit", "status": result["status"], "text": clean_crzp(result["answer"]), "rows": rows,
"urls": result.get("source_urls", []), "crzp_token": token,
"people": [(_names(r.get("authors", [])), crzp.university(r)) for r in rows]}
def _kind(row) -> str:
from tools.crzp.answering.answer import _kind as kind
return kind(row)
def _names(values) -> str:
from tools.crzp.source.parse import display_name
return ", ".join(display_name(v) for v in values) or "neuvedené"
def _work(row) -> str:
return f"{_kind(row)} {row.get('year') or ''}".strip() + f" „{row['title']}“"
def crzp_methods(person: str, row: dict, detail: dict | None) -> str:
""""Ján Pták vo svojej bakalárskej práci „…“ (2025) využíva: ESP32, MQTT, InfluxDB, Home Assistant." —
the work's keywords and the tools its abstract names."""
from tools.answers.content_selection import named_tools
record = detail or row
abstract = record.get("abstract") or ""
found = []
for keyword in record.get("keywords", []):
spelled = re.search(r"(?i)(?<!\w)" + re.escape(keyword) + r"(?!\w)", abstract)
name = spelled.group(0) if spelled else keyword
if named_tools(name) or re.search(r"[A-Z]", name[1:] if spelled else name): # "Home Assistant", not "časová značka"
found.append(name)
found += [t.rstrip(".,;:") for t in named_tools(abstract)]
tools = []
for name in found:
key = normalized(name)
if key and len(key) > 1 and not any(f" {key} " in f" {normalized(t)} " for t in tools):
tools = [t for t in tools if f" {normalized(t)} " not in f" {key} "]
tools.append(name)
where = {"bachelor_thesis": "vo svojej bakalárskej práci", "diploma_thesis": "vo svojej diplomovej práci",
"dissertation": "vo svojej dizertačnej práci"}.get(row.get("work_type"), "vo svojej práci")
return f"{person} {where} „{row['title']}“ ({row.get('year')}) využíva: {', '.join(tools[:12])}."
def remember(rows: list[dict]) -> None:
"""Project the used CRZP works into Neo4j, linked to ZP Wiki people and topics (in the background)."""
from tools.crzp.storage import graph
rows = [r for r in rows if r.get("sid")]
if rows and os.getenv("CRZP_GRAPH_LINKS", "1") != "0":
_pool().submit(graph.project, rows)
def crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None:
"""The CRZP contribution: {"mode", "text", "rows", "people", "works"} or None."""
part = _crzp_part(question, person, complete=complete, student=student)
if part:
remember(part.get("rows") or [])
return part
def _crzp_part(question: str, person: str | None, *, complete=None, student: bool = True) -> dict | None:
from tools.crzp.answering import answer as crzp
from tools.crzp.source.client import CrzpUnavailable
try:
planned = crzp_plan(question, person, complete, student)
if planned is None:
return None
mode, query = planned
low = normalized(question)
if mode == "similar":
return crzp_similar(person, complete=complete)
if mode == "explicit":
return explicit_part(question, crzp._answer_query(question, query, complete))
found, rows = crzp.find(query)
if not rows:
return None
rows = crzp.ranked(question, query, rows)
if mode == "topic":
rows = rows[:5]
lines = [f"- {_names(r.get('authors', []))} — {crzp.university(r)}: „{r['title']}“ ({_kind(r)}, {r.get('year')})"
for r in rows]
return {"mode": mode, "text": "Práce na túto tému v CRZP (najbližšie podľa obsahu):\n\n" + "\n".join(lines),
"rows": rows, "urls": [r["url"] for r in rows if r.get("url")],
"people": [(_names(r.get("authors", [])), crzp.university(r)) for r in rows]}
# person: that person's works at TUKE
rows = [r for r in rows if crzp.university(r) == TUKE]
if not rows:
return None
if not student: # a supervisor: the works he or she led, newest first
rows = sorted(rows, key=lambda r: -(r.get("year") or 0))
lines = [f"- {_names(r.get('authors', []))} — „{r['title']}“ ({_kind(r)}, {r.get('year')})" for r in rows[:8]]
more = f"\n\nSpolu {len(rows)} prác v CRZP na TUKE; zobrazených je 8 najnovších." if len(rows) > 8 else ""
return {"mode": mode, "text": f"Práce, ktoré {person} viedol alebo oponoval, v CRZP (TUKE):\n\n"
+ "\n".join(lines) + more, "rows": rows[:8], "urls": [r["url"] for r in rows[:8] if r.get("url")],
"people": [(_names(r.get("authors", [])), crzp.university(r)) for r in rows[:8]]}
lines, details = [], {}
for row in rows[:3]:
if ABOUT.search(low) or METHODS.search(low):
details[row.get("sid")] = crzp.fetch.detail(row["sid"]) if row.get("sid") else None
detail = details.get(row.get("sid"))
if METHODS.search(low):
lines.append(crzp_methods(person, row, detail))
elif ABOUT.search(low) and detail:
label, text = crzp.summary(detail, complete)
lines.append(f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{text}")
else:
lines.append(f"- {_work(row)} — vedúci {_names(row.get('supervisors', []))}; "
f"oponent {_names(row.get('opponents', []))}")
return {"mode": mode, "text": "\n\n".join(lines), "rows": rows[:3],
"urls": [r["url"] for r in rows[:3] if r.get("url")], "people": [],
"works": [{"source": "crzp", "label": _work(r), "type": r.get("work_type"), "key": crzp.work_key(r)}
for r in rows[:3]]}
except CrzpUnavailable:
return None
def crzp_similar(person: str, *, complete=None) -> dict | None:
"""CRZP works sharing the person's specific tools/topics (searched in CRZP by them)."""
from tools.crzp import fetch
from tools.crzp.answering import answer as crzp
from tools.crzp.source.parse import fold
terms = wiki_terms(person)
if not terms:
return None
scored: dict = {}
for term in terms:
for row in fetch.search(term)["rows"]:
if any(fold(person.split()[-1]) in fold(a) for a in row.get("authors", [])):
continue
text = fold(row["title"] + " " + " ".join(row.get("keywords", [])))
shared = [t for t in terms if fold(t) in text]
if shared:
key = crzp.work_key(row)
scored[key] = (row, sorted(set(shared) | set(scored.get(key, (None, []))[1])))
if not scored:
return None
candidates = list(scored.values())
from tools.retrieval import rerank
profile = wiki_profile(person)
order = rerank.scores(profile, [r["title"] + ". " + ", ".join(r.get("keywords", [])) for r, _ in candidates]) \
or [0] * len(candidates)
# Closest by content first; more shared topics/tools break ties.
best = [c for _, c in sorted(zip(order, candidates), key=lambda i: (-i[0], -len(i[1][1])))][:5]
lines = [f"- {_names(r.get('authors', []))} — {crzp.university(r)}: „{r['title']}“ ({_kind(r)}, {r.get('year')}); "
f"spoločné: {', '.join(shared)}" for r, shared in best]
return {"mode": "similar", "text": "Podobné práce v CRZP (spoločné témy a nástroje):\n\n" + "\n".join(lines),
"rows": [r for r, _ in best], "urls": [r["url"] for r, _ in best if r.get("url")],
"people": [(_names(r.get("authors", [])), crzp.university(r)) for r, _ in best]}
# ------------------------------------------------------------------ composition
def _wiki_people(result: dict) -> list[tuple[str, str]]:
names = re.findall(r"^\*\*([^*\n—]+?)(?:\s+—[^*\n]*)?\*\*\s*$", result.get("answer") or "", re.M)
return [(n.strip(), WIKI_SCHOOL) for n in dict.fromkeys(names) if len(n.split()) >= 2]
def _schools_answer(question: str, engine: str, people: list) -> dict:
lines = [f"- {name} — {school}" for name, school in people]
text = "Školy študentov z predchádzajúcej odpovede:\n\n" + "\n".join(lines)
return _ok(question, engine, text, [], {})
def _ok(question, engine, text, urls, base) -> dict:
keep = {k: v for k, v in base.items() if k not in ("answer", "status", "reason", "delivery_check",
"source_urls", "conversation_context")}
return {**keep, "status": "ok", "reason": None, "accepted_for_delivery": True, "answer": text,
"source_urls": list(dict.fromkeys(urls)), "claims": base.get("claims", []),
"index_version": base.get("index_version"), "complete": base.get("complete", True),
"truncated": base.get("truncated", False), "truncation_reasons": base.get("truncation_reasons", []),
"delivery_check": {"layer": "delivery", "code": "complete", "request_fingerprint": fingerprint(question, engine)}}
def _with_token(result: dict, *, wiki_context=None, people=(), options=(), question="", crzp=None,
primary="wiki") -> dict:
wiki_context = wiki_context or {}
token = seal({"wiki": wiki_context.get("token"), "people": list(people)[:30], "options": list(options)[:6],
"question": question, "crzp": crzp, "primary": primary})
if token:
result["conversation_context"] = {**{k: v for k, v in wiki_context.items() if k != "token"},
"token": token, "work_options": wiki_context.get("work_options", []),
"meeting_options": wiki_context.get("meeting_options", [])}
return result
def answer(question: str, *, engine: str, limit: int = 5, context_token: str | None = None, complete=None,
deliver=None) -> dict:
"""deliver: the ZP Wiki answer function (the API passes its own, tests replace it)."""
from tools.answers.answer_delivery import deliver_answer as default_deliver
from tools.answers.finalize import finalize
deliver_answer = deliver or default_deliver
from tools.crzp.answering.wiki_bridge import wiki_person
question = question.strip()
previous = unseal(context_token)
if previous and previous.get("error"):
from tools.answers.answer_errors import public_failure
failure = public_failure({"reason": previous["error"], "engine": engine})
failure["delivery_check"] = {"layer": "context", "code": "unavailable",
"request_fingerprint": fingerprint(question, engine)}
return failure
wiki_token = previous.get("wiki") if previous else context_token
low = normalized(question)
if previous and previous.get("people") and SCHOOLS.search(low) and not wiki_person(question):
return _with_token(_schools_answer(question, engine, previous["people"]), people=previous["people"],
question=question)
if previous and previous.get("options"):
chosen = _choice(low, previous["options"])
if chosen is not None:
return _answer_choice(previous["question"], chosen, engine=engine, limit=limit, complete=complete,
deliver=deliver_answer)
from tools.crzp.answering.wiki_bridge import wiki_match
match = wiki_match(question)
if previous and previous.get("primary") == "crzp" and previous.get("crzp") and not match:
# The last answer came from CRZP ("O čom je práca Jána Nováka?" -> "2", "a kto bol oponent?").
from tools.crzp.answering import answer as crzp
followed = crzp._answer(question, complete, context_token=previous["crzp"])
rows = followed.pop("_shown", None) or []
followed.pop("_asked", None), followed.pop("_options", None)
if followed.get("accepted_for_delivery") is True:
urls = followed.get("source_urls") or []
text = clean_crzp(followed["answer"]) + ("\n\nZdroje CRZP: " + ", ".join(urls[:8]) if urls else "")
result = {**followed, "answer": text, "delivery_check": {**followed["delivery_check"],
"request_fingerprint": fingerprint(question, engine)}}
token = (followed.get("conversation_context") or {}).get("token")
result.pop("conversation_context", None)
return _with_token(result, people=previous.get("people", []), question=question,
crzp=token or previous["crzp"], primary="crzp")
person, student = (match["name"], match["student"]) if match else (None, True)
started = time.monotonic()
pool = _pool()
try:
wiki_future = pool.submit(deliver_answer, question, engine=engine, limit=limit,
**({"context_token": wiki_token} if wiki_token else {}))
crzp_future = pool.submit(crzp_part, question, person, complete=complete, student=student) if enabled() else None
wiki = wiki_future.result()
try:
part = crzp_future.result(timeout=max(1.0, CRZP_SECONDS - (time.monotonic() - started))) if crzp_future else None
except FutureTimeout:
part = None # CRZP too slow this time: the wiki answer is not held back (its search stays cached)
except Exception:
part = None
finally:
pass # the shared pool stays; a slow CRZP part finishes in it and its search stays cached
if wiki.get("reason") in REFUSED or wiki.get("status") == "ambiguous" and (
person or wiki.get("reason") not in WIKI_CLARIFICATION_YIELDS):
part = None # refused/out of scope, or the wiki asks first (which person, which work): no CRZP
wiki_ok = wiki.get("accepted_for_delivery") is True and wiki.get("status") == "ok"
if part is not None and part["mode"] == "explicit" and wiki.get("status") == "ambiguous":
wiki_ok = False
if wiki.get("accepted_for_delivery") is True and isinstance(wiki.get("answer"), str):
wiki = {**wiki, "answer": finalize(wiki["answer"], wiki.get("source_urls"))}
wiki_context = wiki.get("conversation_context")
if part is None or wiki.get("status") == "ambiguous" and part["mode"] != "explicit":
# (a wiki clarification such as "Uveď meno študenta" yields to a question CRZP answers as a whole)
people = _wiki_people(wiki) if wiki_ok else []
return _with_token(wiki, wiki_context=wiki_context, people=people, question=question) if wiki.get(
"accepted_for_delivery") is True else wiki
# Different works in the two sources and the question names none: ask which one.
if person and part["mode"] == "person" and (METHODS.search(low) or ABOUT.search(low)):
options = _work_options(person, part, low)
if options:
text = (f"{person} má v podkladoch viac prác. Ktorú myslíš?\n\n" +
"\n".join(f"{i}. {o['label']} ({'ZP Wiki' if o['source'] == 'wiki' else 'CRZP'})"
for i, o in enumerate(options, 1)) + "\n\nOdpovedz číslom alebo typom práce.")
result = _ok(question, engine, text, [], wiki)
result.update(status="ambiguous", reason="clarification_required",
delivery_check={"layer": "planning", "code": "clarification_required",
"request_fingerprint": fingerprint(question, engine)})
return _with_token(result, wiki_context=wiki_context, options=options, question=question)
if part["mode"] == "explicit" and part.get("status") == "ambiguous": # CRZP asks which namesake
result = _ok(question, engine, part["text"], [], wiki if wiki_ok else {"index_version": wiki.get("index_version")})
result.update(status="ambiguous", reason="clarification_required",
delivery_check={"layer": "planning", "code": "clarification_required",
"request_fingerprint": fingerprint(question, engine)})
return _with_token(result, question=question, crzp=part.get("crzp_token"), primary="crzp")
crzp_text = "**V CRZP** (Centrálny register záverečných prác):\n\n" + part["text"]
if not wiki_ok and (part["mode"] == "explicit" or wiki.get("status") == "error"):
text, urls = part["text"], part["urls"]
elif wiki_ok:
text = wiki["answer"] + "\n\n" + crzp_text
urls = [*(wiki.get("source_urls") or []), *part["urls"]]
else:
text, urls = "V ZP Wiki k tomu nie je záznam.\n\n" + crzp_text, part["urls"]
if part["urls"]:
text += "\n\nZdroje CRZP: " + ", ".join(part["urls"][:8])
result = _ok(question, engine, text, urls, wiki if wiki_ok else {"index_version": wiki.get("index_version")})
people = (_wiki_people(wiki) if wiki_ok else []) + list(part.get("people") or [])
return _with_token(result, wiki_context=wiki_context, people=people, question=question,
crzp=part.get("crzp_token"), primary="crzp" if part["mode"] == "explicit" and not wiki_ok else "wiki")
def _work_options(person: str, part: dict, low: str) -> list[dict]:
"""The person's works when the two sources hold works of different types and the question names none."""
if re.search(r"\b(?:bakalar\w*|diplom\w*|dizerta\w*|projekt\w*)\b", low):
return []
wiki = wiki_works(person)
wiki_types = {w["type"] for w in wiki.values()}
thesis = {"diploma_thesis": "diplom", "bachelor_thesis": "bakalar"}
crzp_only = [w for w in part.get("works", []) if w["type"] in thesis and w["type"] not in wiki_types]
if not crzp_only:
return []
wiki_options = [{"source": "wiki", "label": label + (f" „{w['topic']}“" if w.get("topic") else ""), "work": label}
for label, w in wiki.items() if w["type"] in thesis][:2]
return wiki_options + crzp_only if wiki_options else []
def _choice(low: str, options: list[dict]) -> dict | None:
number = re.fullmatch(r"\s*(?:cislo\s+)?(\d)\s*[.)]?\s*", low)
if number and 1 <= int(number.group(1)) <= len(options):
return options[int(number.group(1)) - 1]
if CHOICE_CRZP.search(low):
return next((o for o in options if o["source"] == "crzp"), None)
if CHOICE_WIKI.search(low):
return next((o for o in options if o["source"] == "wiki"), None)
return None
def _answer_choice(question: str, chosen: dict, *, engine: str, limit: int, complete=None, deliver=None) -> dict:
from tools.answers.finalize import finalize
deliver_answer = deliver
from tools.crzp.answering.wiki_bridge import wiki_person
if chosen["source"] == "wiki":
adjective = {"diplom": "diplomovej", "bakal": "bakalárskej"}.get(normalized(chosen["work"])[:6].rstrip("ao"), "")
adjective = "diplomovej" if normalized(chosen["work"]).startswith("diplom") else \
"bakalárskej" if normalized(chosen["work"]).startswith("bakal") else adjective
asked = re.sub(r"\b(svojej\s+|jeho\s+|jej\s+)?(práci|práce|prácu)\b", lambda m: (m.group(1) or "") + adjective + " " + m.group(2),
question, count=1) if adjective else question
if asked == question:
asked = question.rstrip(" ?.") + f" ({chosen['work']})?"
result = deliver_answer(asked, engine=engine, limit=limit)
if result.get("accepted_for_delivery") is True and isinstance(result.get("answer"), str):
result = {**result, "answer": finalize(result["answer"], result.get("source_urls"))}
result["delivery_check"] = {**(result.get("delivery_check") or {}),
"request_fingerprint": fingerprint(question, engine)}
return result
person = wiki_person(question)
part = crzp_part(question, person, complete=complete)
rows = [r for r in (part or {}).get("rows", []) if chosen.get("key") and _key_of(r) == chosen["key"]]
if not part or not rows:
return _ok(question, engine, "Vybranú prácu sa v CRZP nepodarilo znova načítať. Skús otázku zopakovať.", [], {})
from tools.crzp.answering import answer as crzp
row = rows[0]
detail = crzp.fetch.detail(row["sid"]) if row.get("sid") else None
low = normalized(question)
if METHODS.search(low):
text = crzp_methods(person, row, detail)
else:
label, summary = crzp.summary(detail, complete) if detail else ("", "")
text = f"**{row['title']}** ({_kind(row)}, {row.get('year')})\n\n{summary}"
urls = [row["url"]] if row.get("url") else []
return _ok(question, engine, text + ("\n\nZdroj: " + urls[0] if urls else ""), urls, {})
def _key_of(row: dict) -> str:
from tools.crzp.answering import answer as crzp
return crzp.work_key(row)

27
tools/answers/content_selection.py
View File

@ -21,6 +21,25 @@ ADMIN = r"\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*\s+(?:text
NOT_COMPLETED = r"\b(?:nie je|nie su|nebol\w*|este nie|zatial nie|nebude|nebudu|nefung\w*|neimplementov\w*|nevytvoren\w*|nevyhodnoten\w*)\b" NOT_COMPLETED = r"\b(?:nie je|nie su|nebol\w*|este nie|zatial nie|nebude|nebudu|nefung\w*|neimplementov\w*|nevytvoren\w*|nevyhodnoten\w*)\b"
# Named tools in a record: acronyms (RAG, MCP), CamelCase (OpenWebUI, GraphRAG), names with digits
# (Neo4J, Fast120, me5mini), domains, and common lower-case tool names.
TOOL_NAMES = set("""python pytorch tensorflow keras docker kubernetes unsloth llamafactory qlora lora optuna whisper
bert slovakbert mistral llama qwen olmo gemma huggingface transformers spacy stanza prodigy rasa langchain llamaindex
elasticsearch opensearch neo4j cypher postgres postgresql sqlite mysql mongodb redis fastapi flask django react vue
angular nodejs javascript typescript java kotlin matlab opencv yolo onnx ollama openwebui telegram discord grafana
prometheus jupyter colab git gitea github gitlab slurm faiss qdrant chroma milvus pandas numpy scikit sklearn skills
skill streamlit gradio langgraph cogdb""".split())
TOOL_PATTERN = re.compile(r"(?<![\w.])(?:[A-Z][A-Za-z]*[A-Z][A-Za-z]*|[A-Za-z]+\d[\w.]*|\d+[A-Za-z][\w]*|[a-z0-9-]+\.[a-z]{2,4})(?![\w])")
SEARCH_METHODS = r"\b(?:vektorov\w* vyhladav\w*|textov\w* vyhladav\w*|vyhladavani\w*|dotrenov\w*|fine.?tun\w*|embedding\w*|grafov\w* databaz\w*|znalostn\w* graf\w*)\b"
def named_tools(text: str) -> list[str]:
"""Tool and technology names the record mentions (not dates, not plain words)."""
found = [m for m in TOOL_PATTERN.findall(text or "") if not re.fullmatch(r"\d+[.:]\d*", m) and len(m) >= 2]
found += [w for w in re.findall(r"[A-Za-z][A-Za-z0-9]+", text or "") if w.lower() in TOOL_NAMES]
return list(dict.fromkeys(found))
def has_qualification(text: str) -> bool: def has_qualification(text: str) -> bool:
return bool( return bool(
re.search( re.search(
@ -136,15 +155,21 @@ def facets_for(text: str, role: str, question: str, phase: str | None = None) ->
r"\b(?:problem\w*|tazkost\w*|tazke|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*|nefung\w*|nepasuj\w*|nesulad\w*|nekompatibil\w*)\b", n r"\b(?:problem\w*|tazkost\w*|tazke|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*|nefung\w*|nepasuj\w*|nesulad\w*|nekompatibil\w*)\b", n
) and role not in {"assignment", "task"}: ) and role not in {"assignment", "task"}:
facets.append("problems") facets.append("problems")
tools = named_tools(text)
if re.search( if re.search(
r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*|indexov\w*|databaz\w*|webov\w* rozhrani\w*|frontend\w*|backend\w*|vyskus\w*|sumarizac\w*)\b", r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*|indexov\w*|databaz\w*|webov\w* rozhrani\w*|frontend\w*|backend\w*|vyskus\w*|sumarizac\w*)\b",
n, n,
): ) or tools or re.search(SEARCH_METHODS, n):
actual_use = actual_methods_requested(question) actual_use = actual_methods_requested(question)
# A status/result record naming a tool ("Sú Skills aj MCP server", "začíname pracovať na
# GraphRAG a Neo4J") states real use unless it is phrased as a plan or as not done.
stated_in_status = (role in {"progress", "result"} and bool(tools or re.search(SEARCH_METHODS, n))
and not re.search(PLANNED, n))
actual_statement = not re.search(NOT_COMPLETED, n) and ( actual_statement = not re.search(NOT_COMPLETED, n) and (
re.search(COMPLETED, n) or re.search(r'\bvyskus(?:any|ana|ane|al|ala|ali)\b',n) re.search(COMPLETED, n) or re.search(r'\bvyskus(?:any|ana|ane|al|ala|ali)\b',n)
or re.search(r"\b(?:pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|vieme indexov\w*)\b", n) or re.search(r"\b(?:pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|vieme indexov\w*)\b", n)
or (not re.search(PLANNED,n) and re.search(r'\b(?:frontend|backend)\s+\w+',n)) or (not re.search(PLANNED,n) and re.search(r'\b(?:frontend|backend)\s+\w+',n))
or stated_in_status
) )
if not actual_use or (role not in {"assignment", "task"} and actual_statement): if not actual_use or (role not in {"assignment", "task"} and actual_statement):
facets.append("methods") facets.append("methods")

17
tools/answers/conversation.py
View File

@ -23,6 +23,19 @@ MAX_TOKEN = 16000
LIFETIME = 24 * 60 * 60 LIFETIME = 24 * 60 * 60
MEETING_REF = re.compile(r'\b(?:vtedy|(?:v|na|o|z|pri) (?:danom|tomto|tom|tomto istom) stretnuti)\b') MEETING_REF = re.compile(r'\b(?:vtedy|(?:v|na|o|z|pri) (?:danom|tomto|tom|tomto istom) stretnuti)\b')
WORK_REF = re.compile(r'\b(?:(?:tej|tu|t[uú]) (?:druhej|druhu|prvej|prvu)|(?:pri|o) druhej) (?:praci|pracu|projekte)\b') WORK_REF = re.compile(r'\b(?:(?:tej|tu|t[uú]) (?:druhej|druhu|prvej|prvu)|(?:pri|o) druhej) (?:praci|pracu|projekte)\b')
# A pronoun points at the previous person ("Kto bol jeho vedúci?", "Aké výsledky dosiahla?" is no pronoun).
PRONOUN_REF = re.compile(r'\b(?:jeho|jej|ho|mu|nemu|nej|nom|onej|on|ona|tento student|tato studentka|ten student|'
r'ta studentka|toho studenta|tej studentky|ten isty|ta ista|daneho studenta|danej studentky|'
r'tohto studenta|tejto studentky)\b')
# A short question made only of these words asks about an aspect of the previous subject
# ("Kto bol vedúci?", "Aké technológie použil?"); any other content word makes it a new question.
ASPECT_WORDS = set("""kto bol bola boli je su bude co aky aka ake akej akom akych ktory ktora ktore kedy kde ako kolko
mal mala mali ma maju robil robila pracoval pracovala pouzil pouzila pouzival pouzivala dosiahol dosiahla urobil
urobila napisal napisala veduci veduceho veducim skolitel skolitela konzultant konzultanta oponent oponenta vysledky
vysledok vysledkoch technologie technologiu nastroje nastroj metody metodu ciel ciele tema temu temy nazov nazvy
praca prace pracu praci rok roku roky stretnutie stretnutia stretnuti posledne prve ulohy ulohu stav pokrok zadanie
plan klucove slova tagy jazyk jazyky data dataset model modely sa si mu jej jeho ho tam v vo na o z zo s so k a aj ale
uz este teraz potom tiez prosim daj ukaz vypis povedz""".split())
BOTH = re.compile(r'^(?:(?:daj|uved|ukaz|vypis|porovnaj|chcem|prosim)\s+)?(?:obe|oba|obidv\w*)(?:\s+(?:prace|projekty))?[.!?]?$') BOTH = re.compile(r'^(?:(?:daj|uved|ukaz|vypis|porovnaj|chcem|prosim)\s+)?(?:obe|oba|obidv\w*)(?:\s+(?:prace|projekty))?[.!?]?$')
@ -163,7 +176,9 @@ def resolve_question(question, *, token=None, catalog, index, version):
explicit_date=bool(re.search(r'\b\d{1,2}\.\s*\d{1,2}\.\s*[12]\d{3}\b',query)) explicit_date=bool(re.search(r'\b\d{1,2}\.\s*\d{1,2}\.\s*[12]\d{3}\b',query))
# An explicit date is stronger than a redundant word such as "vtedy". # An explicit date is stronger than a redundant word such as "vtedy".
needs_context=meeting_ref and not explicit_date or bool(work_ref) or both needs_context=meeting_ref and not explicit_date or bool(work_ref) or both
short_followup=bool(context and (explicit_date or re.match(r'^(?:a |ale |čo |co |aké |ake |kedy |čo sa |co sa )',n))) aspect_only=bool(n) and len(n.split())<=7 and all(w in ASPECT_WORDS or w.isdigit() for w in n.split())
short_followup=bool(context and (explicit_date or re.match(r'^(?:a |ale |čo |co |aké |ake |kedy |čo sa |co sa )',n)
or PRONOUN_REF.search(n) or aspect_only))
if needs_context and context is None: if needs_context and context is None:
return ResolvedQuestion(query,clarification=( return ResolvedQuestion(query,clarification=(
'Ktoré stretnutie máš na mysli? Uveď študenta a dátum alebo presné označenie stretnutia.' if meeting_ref 'Ktoré stretnutie máš na mysli? Uveď študenta a dátum alebo presné označenie stretnutia.' if meeting_ref

113
tools/answers/finalize.py Normal file
View File

@ -0,0 +1,113 @@
"""Last presentation pass over every accepted ZP Wiki answer (RAG and GraphRAG).
It changes wording only, never facts:
- sources are written as full links, never as numbers ("[1]");
- notices that explain the system instead of answering ("Vedúci je určený
podľa autora profilovej stránky…", "Profil obsahuje aj záznamy bez úplného
dátumu…", "Rok nedopĺňam…") are dropped;
- "this detail is not recorded" lines stay only when the answer has nothing
else; then one plain sentence says so;
- technical labels ("Názov / téma zo zdroja:") become plain words.
"""
from __future__ import annotations
import re
DROP = [
r"Vedúci je určený podľa autora profilovej stránky[^\n]*",
r"Profil obsahuje aj záznamy bez úplného dátumu[^\n]*",
r"Rok nedopĺňam z roku práce\.?",
r"Údaje sú z indexovan[^\n]*",
r"Indexovan[éý][^\n]*",
r"Zobrazený je obmedzený výber[^\n]*",
]
PHASE_TAG = re.compile(r"\s+\((?:zaznamenaný výsledok|priebežný stav|plán|zadanie|podklad|navrhovaný cieľ|téma alebo názov|"
r"úloha označená ako splnená)\)(?=\s*$)", re.M)
# Students whose only match is an assignment/plan or reading: one line at the end instead of a block each.
SIDE = {"téma je v zadaní, téme alebo pláne práce; zaznamenaný výsledok k nej sa nenašiel.":
"Téma je len v zadaní alebo pláne, bez zaznamenaného výsledku",
"záznamy dokladajú len štúdium alebo poznámky k téme, nie vlastnú realizáciu.":
"Len štúdium alebo poznámky k téme"}
BULLET = re.compile(r"^- (?P<work>[^;\n:]+?); (?P<phase>[^;:\n]+?)(?:; záznam (?P<date>[^:\n]+?))?: (?P<text>.*)$", re.M)
LABEL_PREFIX = re.compile(r"^(?:-\s+|\*(?:Názov|Meno)[^*]*\*:\s*|(?:Názov|Téma)\s*:\s*)+")
def _bullet(match: re.Match) -> str:
text = LABEL_PREFIX.sub("", match.group("text").strip()).strip()
if not text or text.startswith("#"):
return "" # a bare heading ("## Diplomový projekt 2") says nothing
if re.fullmatch(r"(?:Vedúci\w*:\s*)?(?:(?:Ing|Mgr|doc|prof|Bc|RNDr)\.\s*)*[A-ZÁ-Ž]\w+\s+[A-ZÁ-Ž]\w+(?:,\s*PhD\.?)?\.?", text):
return "" # only the supervisor's name, already the subject of the answer
date = (match.group("date") or "").replace("(rok neuvedený)", "").strip(" :()")
return f"- {match.group('work').strip()}" + (f" ({date})" if date else "") + f": {text}"
def _collapse_side_people(blocks: list[str]) -> list[str]:
groups, current = [], []
for block in blocks:
if re.fullmatch(r"\*\*[^*\n]+\*\*", block) and current:
groups.append(current)
current = []
current.append(block)
if current:
groups.append(current)
kept, side = [], {}
for group in groups:
head = re.fullmatch(r"\*\*([^*\n]+)\*\*", group[0])
label = next((v for k, v in SIDE.items() for b in group if b.endswith(k)), None)
if head and label:
side.setdefault(label, []).append(head.group(1))
else:
kept += group
return kept + [f"{label}: {', '.join(names)}." for label, names in side.items()]
MISSING = re.compile(r"(?:požadovaný údaj nie je v tomto zápise doložený|v podkladoch chýbajú doložené údaje|"
r"Požadované údaje k tomuto stretnutiu nie sú v dostupnom výbere doložené)", re.I)
RENAME = [
(r"Názov / téma zo zdroja:\s*(?:\*Názov [^*]*\*:\s*)?", "Téma: "),
(r"Vedúci práce \(autor profilu študenta\):", "Vedúci práce:"),
(r"Konzultant práce \(zo zdroja\):", "Konzultant:"),
]
INLINE_CITE = re.compile(r"\s*(?:\[\d+\])+(?!\(|:)")
REFERENCE_LINE = re.compile(r"^\[\d+\]:\s*\S+\s*$", re.M)
SOURCES = re.compile(r"^Zdroj(?:e)?:\s*(.+)$", re.M)
def _links(line: str) -> list[str]:
urls = re.findall(r"\]\((https?://[^)\s]+)\)", line) or re.findall(r"https?://[^\s,)\]]+", line)
return list(dict.fromkeys(urls))
def finalize(text: str, source_urls: list[str] | None = None) -> str:
if not isinstance(text, str) or not text.strip():
return text
out = REFERENCE_LINE.sub("", text)
sources, label = [], None
for match in SOURCES.finditer(out):
sources += _links(match.group(1))
label = label or match.group(0).split(":", 1)[0]
out = SOURCES.sub("", out)
out = INLINE_CITE.sub("", out)
for pattern in DROP:
out = re.sub(pattern, "", out)
for pattern, replacement in RENAME:
out = re.sub(pattern, replacement, out)
out = PHASE_TAG.sub("", out)
out = BULLET.sub(_bullet, out)
out = re.sub(r"^- Téma alebo názov doložený pri tej istej práci:\s*(?:\*[^*]*\*:\s*)?", "- Téma: ", out, flags=re.M)
out = re.sub(r"(?m)^(\s*- [^:\n]+:)\s+-\s+", r"\1 ", out) # "Diplomový projekt: - Rozpoznávanie…'
blocks = _collapse_side_people([b.strip() for b in re.split(r"\n\s*\n", out) if b.strip()])
substantive = [b for b in blocks if not MISSING.search(b) and not re.fullmatch(r"\*\*[^*]+\*\*", b)
and not re.fullmatch(r"(?:\*\*[^*]+\*\*\s*)?(?:Posledný záznam[^\n]*|Zadanie a ciele:)", b)]
if substantive:
blocks = [b for b in blocks if not MISSING.search(b)]
blocks = [b for i, b in enumerate(blocks) # a heading left without its content
if not (re.fullmatch(r"[A-ZÁ-Ž][^\n]{0,40}:", b) and (i + 1 == len(blocks) or blocks[i + 1].startswith("**")))]
else:
heads = [b for b in blocks if re.fullmatch(r"\*\*[^*]+\*\*", b) or b.startswith("**")]
blocks = heads[:1] + ["ZP Wiki k tomu nemá záznam."]
sources = list(dict.fromkeys(sources)) # rewrite the answer's own source line, never add one
if sources:
blocks.append((label or "Zdroje") + ": " + ", ".join(sources))
return "\n\n".join(blocks)

4
tools/answers/question_intent.py
View File

@ -7,7 +7,7 @@ FACET_PATTERNS = {
"assignment": r"\b(?:zadani\w*|ciel\w*)\b|\bco\s+(?:ma|maju|mal[aoiy]?)\b[^?;!.]*\b(?:vytvorit|vypracovat|implementovat|navrhnut|zostavit)\b", "assignment": r"\b(?:zadani\w*|ciel\w*)\b|\bco\s+(?:ma|maju|mal[aoiy]?)\b[^?;!.]*\b(?:vytvorit|vypracovat|implementovat|navrhnut|zostavit)\b",
"plans": r"\b(?:planovan\w*|dalsi\w*\s+uloh\w*|uloh\w*|co\s+(?:ma|maju|mal[aoiy]?)\s+(?:urobit|spravit)(?:\s+dalej)?)\b", "plans": r"\b(?:planovan\w*|dalsi\w*\s+uloh\w*|uloh\w*|co\s+(?:ma|maju|mal[aoiy]?)\s+(?:urobit|spravit)(?:\s+dalej)?)\b",
"problems": r"\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b", "problems": r"\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b",
"methods": r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*|pristup\w*|frontend\w*|backend\w*)\b", "methods": r"\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*|vyuziv\w*|vyuzil\w*|pristup\w*|frontend\w*|backend\w*)\b",
"experiments": r"\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*|vyhodnot\w*)\b", "experiments": r"\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*|vyhodnot\w*)\b",
"progress": r"\b(?:priebezn\w*\s+)?stav\w*\b", "progress": r"\b(?:priebezn\w*\s+)?stav\w*\b",
"results": r"\b(?:vysledk\w*|dosiahn\w*|funkcnost\w*)\b", "results": r"\b(?:vysledk\w*|dosiahn\w*|funkcnost\w*)\b",
@ -144,7 +144,7 @@ def facet_mode(question: str) -> bool:
def actual_methods_requested(question: str) -> bool: def actual_methods_requested(question: str) -> bool:
return bool( return bool(
re.search( re.search(
r"\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*|vyskus(?:al|ala|ali)\w*)\b|" r"\b(?:skutocne|pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|implementoval\w*|vyskus(?:al|ala|ali)\w*)\b|"
r"\bake technolog\w* ma\b|" r"\bake technolog\w* ma\b|"
r"\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b", r"\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b",
normalized(question), normalized(question),

5
tools/answers/question_plan.py
View File

@ -174,7 +174,10 @@ def direct_spec(question, resolution, *, meeting=None, route=None, relative_year
meeting=meeting.model_copy(update={'count':count}) meeting=meeting.model_copy(update={'count':count})
facets=requested_facets(q) facets=requested_facets(q)
if route is not None and route.facets: if route is not None and route.facets:
facets=list(route.facets) # A meeting question asks for the whole meeting ("čo na ňom riešil") unless it names an
# aspect itself; the classifier must not narrow it to e.g. results it guessed.
# Aspects the question names itself ("aké úlohy dostal" -> plans) win over the classifier's guess.
facets=[f for f in route.facets if f in facets] if route.type=='meeting' else (facets or list(route.facets))
global_discovery=discovery_question(q) global_discovery=discovery_question(q)
if route is None and mode not in {'similar','supervisor','advisors','completion','meeting','period','proposal'} and global_discovery: if route is None and mode not in {'similar','supervisor','advisors','completion','meeting','period','proposal'} and global_discovery:
mode='discovery' mode='discovery'

7
tools/answers/synthesis.py
View File

@ -489,9 +489,12 @@ def render(draft, rows, result, plan):
bullets=bool(re.search(r'\bbod\w*\b',normalized(plan.question))) or plan.presentation=='list' and plan.intent!='meeting' bullets=bool(re.search(r'\bbod\w*\b',normalized(plan.question))) or plan.presentation=='list' and plan.intent!='meeting'
text=('\n' if bullets else ' ').join(dict.fromkeys(('- ' if bullets else '')+c.text+' '+cites(c) for c in claims)) text=('\n' if bullets else ' ').join(dict.fromkeys(('- ' if bullets else '')+c.text+' '+cites(c) for c in claims))
dates=' (dátum záznamu: '+displayed+')' if displayed and plan.intent!='meeting' else '' dates=' (dátum záznamu: '+displayed+')' if displayed and plan.intent!='meeting' else ''
title=({'result':'Výsledky','progress':'Priebeh a zaznamenané ťažkosti', title=({'result':'Výsledky','progress':'Stav',
'assignment':'Zadanie','planned':'Ďalší postup','unknown':'Doplňujúce podklady'}.get(phase,PHASES[phase]) 'assignment':'Zadanie','planned':'Úlohy','unknown':'Ďalšie poznámky'}.get(phase,PHASES[phase])
if plan.intent=='meeting' else PHASES[phase]) if plan.intent=='meeting' else PHASES[phase])
if phase=='planned' and list(plan.facets)==['plans']: # "A aké úlohy dostal?"
title='Úlohy'+(' zo stretnutia '+displayed if displayed else '');dates='';bullets=True
text='\n'.join(dict.fromkeys('- '+c.text+' '+cites(c) for c in claims))
sections.append(title+dates+(':\n' if bullets else ': ')+text) sections.append(title+dates+(':\n' if bullets else ': ')+text)
cell_dates=' (dátum záznamu: '+displayed+')' if displayed else '' cell_dates=' (dátum záznamu: '+displayed+')' if displayed else ''
table_cells[(scope,label,phase)].append((cell_dates+': ' if cell_dates else '')+text) table_cells[(scope,label,phase)].append((cell_dates+': ' if cell_dates else '')+text)

46
tools/claude/build_claude_packages.py
View File

@ -1,5 +1,13 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Build reproducible, self-contained Claude skill ZIPs without runtime data or secrets.""" """Build the self-contained Claude skills without runtime data or secrets.
The skills live unpacked in packages/claude/dist/<engine>/ (SKILL.md, scripts,
manifest); no archive is kept in the repository. A reproducible ZIP for upload
to claude.ai is made only on request:
python tools/claude/build_claude_packages.py # refresh dist/<engine>/
python tools/claude/build_claude_packages.py --zip /tmp/x # also /tmp/x/zpwiki-<engine>.zip
"""
from __future__ import annotations from __future__ import annotations
@ -11,18 +19,18 @@ import zipfile
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[2]
BASE = ROOT / "packages/claude" BASE = ROOT / "packages/claude"
# Skill sources live unpacked next to their archives: dist/<engine>/ holds # Skill sources live unpacked: dist/<engine>/ (SKILL.md, scripts, manifest).
# exactly the files of dist/zpwiki-<engine>.zip (SKILL.md, scripts, manifest).
SOURCES = BASE / "dist" SOURCES = BASE / "dist"
ENGINES = ("rag", "graphrag", "crzp") ENGINES = ("rag", "graphrag")
PACKAGE_VERSION = "1.13.0" PACKAGE_VERSION = "2.0.0"
def json_bytes(value): def json_bytes(value):
return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode() return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode()
def build_package(engine: str, output: Path) -> dict: def build_package(engine: str, output: Path, archive: Path | None = None) -> dict:
"""Write output/<engine>/; with archive, also archive/zpwiki-<engine>.zip."""
if engine not in ENGINES: if engine not in ENGINES:
raise ValueError("Unknown engine") raise ValueError("Unknown engine")
name = engine name = engine
@ -49,36 +57,38 @@ def build_package(engine: str, output: Path) -> dict:
} }
) )
output.mkdir(parents=True, exist_ok=True) output.mkdir(parents=True, exist_ok=True)
# The unpacked folder is kept identical to the archive (shared client.py # Shared client.py and the manifest are refreshed from their single source on every build.
# and the manifest are refreshed from their single source on every build).
folder = output / name folder = output / name
for path, data in files.items(): for path, data in files.items():
target = folder / path target = folder / path
target.parent.mkdir(parents=True, exist_ok=True) target.parent.mkdir(parents=True, exist_ok=True)
if not target.exists() or target.read_bytes() != data: if not target.exists() or target.read_bytes() != data:
target.write_bytes(data) target.write_bytes(data)
destination = output / ("zpwiki-" + name + ".zip") result = {"name": name, "folder": name,
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as archive: "manifest_sha256": hashlib.sha256(files["manifest.json"]).hexdigest()}
if archive is None:
return result
archive.mkdir(parents=True, exist_ok=True)
destination = archive / ("zpwiki-" + name + ".zip")
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as bundle:
for path, data in sorted(files.items()): for path, data in sorted(files.items()):
info = zipfile.ZipInfo(name + "/" + path, date_time=(1980, 1, 1, 0, 0, 0)) info = zipfile.ZipInfo(name + "/" + path, date_time=(1980, 1, 1, 0, 0, 0))
info.compress_type = zipfile.ZIP_DEFLATED info.compress_type = zipfile.ZIP_DEFLATED
info.create_system = 3 info.create_system = 3
info.external_attr = 0o100644 << 16 info.external_attr = 0o100644 << 16
archive.writestr(info, data) bundle.writestr(info, data)
return { return {**result, "archive": destination.name, "sha256": hashlib.sha256(destination.read_bytes()).hexdigest()}
"name": name,
"archive": destination.name,
"sha256": hashlib.sha256(destination.read_bytes()).hexdigest(),
}
def main(): def main():
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, default=BASE / "dist") parser.add_argument("--output", type=Path, default=BASE / "dist")
parser.add_argument("--zip", type=Path, help="also write zpwiki-<engine>.zip for upload into this folder")
args = parser.parse_args() args = parser.parse_args()
packages = [build_package(engine, args.output) for engine in ENGINES] packages = [build_package(engine, args.output, args.zip) for engine in ENGINES]
(args.output / "packages.json").write_bytes( (args.output / "packages.json").write_bytes(
json_bytes({"schema_version": 1, "packages": packages}) json_bytes({"schema_version": 2, "packages": [{k: p[k] for k in ("name", "folder", "manifest_sha256")}
for p in packages]})
) )
print(json.dumps(packages, ensure_ascii=False)) print(json.dumps(packages, ensure_ascii=False))

239
tools/crzp/answering/answer.py
View File

@ -61,6 +61,8 @@ def keep(row: dict, query: CrzpQuery) -> bool:
return False # follow-up about chosen works of the previous answer return False # follow-up about chosen works of the previous answer
if query.school_name and university(row) != query.school_name: if query.school_name and university(row) != query.school_name:
return False # the chosen namesake return False # the chosen namesake
if query.works and work_key(row) not in query.works:
return False # the chosen namesake's works (two students of one name at one university)
if any(not people_in_role(row, p.name, p.role) for p in query.people): if any(not people_in_role(row, p.name, p.role) for p in query.people):
return False return False
if query.years and row.get("year") not in query.years: if query.years and row.get("year") not in query.years:
@ -147,20 +149,70 @@ def prefer_exact(rows: list[dict], query: CrzpQuery) -> list[dict]:
return rows return rows
LEVEL = {"bachelor_thesis": 1, "diploma_thesis": 2, "doctoral_thesis": 3, "dissertation": 3, "habilitation_thesis": 4}
def work_key(row: dict) -> str:
return f"{' '.join(fold(row['title']).rstrip(' .').split())}|{row.get('year')}"
def _split_students(group: dict) -> list[dict]:
"""One student writes one work of each kind: two bachelor theses of "Ján Novák" at one
university are two persons. Works are given, oldest first, to the first person who
has no work of that kind yet and whose last work is at most 8 years older."""
persons = []
for row in sorted(group["rows"], key=lambda r: r.get("year") or 0):
kind, year = row.get("work_type"), row.get("year") or 0
target = next((p for p in persons if kind not in p["kinds"] and year - p["last"] <= 8), None)
if target is None:
target = {"kinds": set(), "last": year, "rows": []}
persons.append(target)
target["kinds"].add(kind)
target["last"] = max(target["last"], year)
target["rows"].append(row)
return [{**group, "rows": p["rows"], "count": len(p["rows"]), "years": {r.get("year") for r in p["rows"]}}
for p in persons]
def identities(rows: list[dict], person) -> list[dict]: def identities(rows: list[dict], person) -> list[dict]:
"""The asked name split into probable persons: the same written name (with diacritics) """The asked name split into probable persons (CRZP has no person identifiers):
at the same university is one person; another spelling or university is another one. - the same written name (with diacritics) at the same university is one person,
CRZP has no person identifiers, so this is the most it can tell.""" another spelling or university another one;
- a name that is only an author there is split when it has two works of one kind
(two bachelor theses are two students);
- an author at another university whose earlier work leads to this one
(bachelor -> diploma within 4 years) is marked as possibly the same person."""
groups: dict = {} groups: dict = {}
seen = set()
for row in rows: for row in rows:
if work_key(row) in seen:
continue # the same work held twice in CRZP
seen.add(work_key(row))
for name in people_in_role(row, person.name, person.role): for name in people_in_role(row, person.name, person.role):
key = (display_name(name), university(row)) key = (display_name(name), university(row))
group = groups.setdefault(key, {"name": key[0], "school": key[1], "count": 0, "years": set(), group = groups.setdefault(key, {"name": key[0], "school": key[1], "count": 0, "years": set(),
"roles": Counter()}) "roles": Counter(), "rows": []})
group["count"] += 1 group["count"] += 1
group["years"].add(row.get("year")) group["years"].add(row.get("year"))
group["rows"].append(row)
group["roles"].update(role for role, field in ROLE_FIELD.items() if name in row.get(field, [])) group["roles"].update(role for role, field in ROLE_FIELD.items() if name in row.get(field, []))
return sorted(groups.values(), key=lambda g: (-g["count"], -max((y or 0) for y in g["years"]), g["name"], g["school"])) result = []
for group in groups.values():
result += _split_students(group) if set(group["roles"]) == {"author"} else [group]
for group in result:
group["hint"] = None
if set(group["roles"]) != {"author"}:
continue
first = min(group["rows"], key=lambda r: r.get("year") or 0)
for other in result:
if other is group or other["name"] != group["name"] or set(other["roles"]) != {"author"}:
continue
last = max(other["rows"], key=lambda r: r.get("year") or 0)
if (other["school"] != group["school"]
and LEVEL.get(last.get("work_type"), 0) < LEVEL.get(first.get("work_type"), 0)
and 0 < (first.get("year") or 0) - (last.get("year") or 0) <= 4):
group["hint"] = f"možno tá istá osoba ako {other['school']} ({last.get('year')})"
return sorted(result, key=lambda g: (-g["count"], -max((y or 0) for y in g["years"]), g["name"], g["school"]))
def _identity_line(group: dict) -> str: def _identity_line(group: dict) -> str:
@ -168,7 +220,8 @@ def _identity_line(group: dict) -> str:
span = (f"{years[0]}–{years[-1]}" if len(years) > 1 else str(years[0])) if years else "rok neuvedený" span = (f"{years[0]}–{years[-1]}" if len(years) > 1 else str(years[0])) if years else "rok neuvedený"
roles = ", ".join(ROLE_LABEL[r] for r, _ in group["roles"].most_common()) roles = ", ".join(ROLE_LABEL[r] for r, _ in group["roles"].most_common())
works = f"{group['count']} {'práca' if group['count'] == 1 else 'práce' if group['count'] < 5 else 'prác'}" works = f"{group['count']} {'práca' if group['count'] == 1 else 'práce' if group['count'] < 5 else 'prác'}"
return f"{group['name']} — {group['school']} ({roles}; {span}; {works})" hint = f"; {group['hint']}" if group.get("hint") else ""
return f"{group['name']} — {group['school']} ({roles}; {span}; {works}{hint})"
def namesakes(rows: list[dict], query: CrzpQuery) -> str: def namesakes(rows: list[dict], query: CrzpQuery) -> str:
@ -185,7 +238,7 @@ def namesakes(rows: list[dict], query: CrzpQuery) -> str:
def clarification(rows: list[dict], query: CrzpQuery) -> tuple[str, list[dict]] | None: def clarification(rows: list[dict], query: CrzpQuery) -> tuple[str, list[dict]] | None:
"""Count, summary, keywords, relations and similar works of one person must not mix """Count, summary, keywords, relations and similar works of one person must not mix
namesakes: ask which person is meant (a list shows all with a notice instead).""" namesakes: ask which person is meant (a list shows all with a notice instead)."""
if not query.people or query.intent in ("list", "schools") or query.school_name or query.sids: if not query.people or query.intent in ("list", "schools") or query.school_name or query.sids or query.works:
return None return None
groups = identities(rows, query.people[0]) groups = identities(rows, query.people[0])
if len(groups) < 2: if len(groups) < 2:
@ -195,21 +248,25 @@ def clarification(rows: list[dict], query: CrzpQuery) -> tuple[str, list[dict]]
"identifikátorom). Ktorú myslíš?\n\n" + "\n".join(f"{i}. {_identity_line(g)}" for i, g in "identifikátorom). Ktorú myslíš?\n\n" + "\n".join(f"{i}. {_identity_line(g)}" for i, g in
enumerate(groups, 1)) + enumerate(groups, 1)) +
"\n\nOdpovedz číslom, školou alebo celým menom.") "\n\nOdpovedz číslom, školou alebo celým menom.")
return text, [{"name": g["name"], "school": g["school"]} for g in groups] return text, [{"name": g["name"], "school": g["school"], "works": [work_key(r) for r in g["rows"]][:30]}
for g in groups]
def _merge(results: list[dict]) -> dict: def _merge(results: list[dict]) -> dict:
rows, seen = [], set() rows, seen = [], {}
for result in results: for result in results:
for row in result["rows"]: for row in result["rows"]:
key = row.get("sid") or (fold(row["title"]), row.get("year"), tuple(row.get("authors", []))) key = (fold(row["title"]).rstrip(" ."), row.get("year"), tuple(row.get("authors", [])))
if key not in seen: if key not in seen:
seen.add(key) seen[key] = row
rows.append(row) rows.append(row)
elif row.get("sid") and not seen[key].get("sid"): # the same work read with its permalink
seen[key].update(sid=row["sid"], url=row.get("url"))
return {"texts": [r["query"] for r in results], "totals": [r["total"] for r in results], return {"texts": [r["query"] for r in results], "totals": [r["total"] for r in results],
"total": results[0]["total"] if len(results) == 1 else sum(r["total"] for r in results), "total": results[0]["total"] if len(results) == 1 else sum(r["total"] for r in results),
"facets": results[0]["facets"], "rows": rows, "complete": all(r["complete"] for r in results), "facets": results[0]["facets"], "rows": rows, "complete": all(r["complete"] for r in results),
"incomplete": [(r["query"], r["total"], len(r["rows"])) for r in results if not r["complete"]]} "incomplete": [(r["query"], r["total"], len(r["rows"])) for r in results if not r["complete"]],
"parts": results}
def _unverified(query: CrzpQuery, text: str) -> str | None: def _unverified(query: CrzpQuery, text: str) -> str | None:
@ -229,22 +286,48 @@ def _used_with_topic(term: str, rows: list[dict]) -> bool:
AGGREGATES = ("count", "keywords", "schools", "relation") AGGREGATES = ("count", "keywords", "schools", "relation")
def find(query: CrzpQuery, *, links: bool = True) -> tuple[dict | None, list[dict]]: def by_facets(query: CrzpQuery) -> bool:
"""A count, or the supervisors of works on a topic, that CRZP computes itself: every
condition is in the search (text, years, work type) and one text per meaning, so the
CRZP total and the full year/type/supervisor facets are exact without reading rows."""
counted = query.intent == "count" or query.intent == "relation" and query.target_role == "supervisor"
return (counted and not query.people and not query.title and not query.sids and not query.school_name
and not query.alternatives and len(query.topics) <= 1)
def reader(query: CrzpQuery):
"""How one search text is read for this question:
- counted by CRZP (by_facets): the first page with the full facets only;
- summaries (tags, schools, opponents, a person's works): every row up to
DEEP_ROWS, in year ranges at once, permalinks only where a work is shown;
- a list of a school or workplace: the newest LINKED_ROWS works, all with permalinks;
- a topic list, a summary of a work, similar works: the LINKED_ROWS most relevant rows."""
window = filters(query)
if by_facets(query):
return lambda text: fetch.search(text, links=False, depth=20, **window)
if query.intent in ("keywords", "schools", "count"):
return lambda text: fetch.search_deep(text, links=False, **window)
if query.intent == "relation" or query.people:
return lambda text: fetch.search_deep(text, links=True, **window)
if query.intent == "list" and (query.school or query.unit) and not query.topics and not query.title:
return lambda text: fetch.search_deep(text, links=True, limit=fetch.LINKED_ROWS, **window)
return lambda text: fetch.search(text, links=query.intent != "count", **window)
def find(query: CrzpQuery) -> tuple[dict | None, list[dict]]:
"""First search group whose rows satisfy all conditions; (merged result, rows). """First search group whose rows satisfy all conditions; (merged result, rows).
A model-proposed alternative term is searched only if CRZP uses it in a work A model-proposed alternative term is searched only if CRZP uses it in a work
already found for the topic, or if nothing was found yet.""" already found for the topic, or if nothing was found yet."""
last, seen = None, [] last, seen = None, []
query.searched = [] query.searched = []
# Counts and summaries read up to DEEP_ROWS rows (no permalinks needed); a person's read = reader(query)
# works are filtered by role in code, so all of them are read as well.
depth = fetch.DEEP_ROWS if query.intent in AGGREGATES or query.people else fetch.LINKED_ROWS
for group in search_groups(query): for group in search_groups(query):
results, verified_rows, unverified = [], 0, [] results, verified_rows, unverified = [], 0, []
for text in group: for text in group:
term = _unverified(query, text) term = _unverified(query, text)
if term and seen and not _used_with_topic(term, seen): if term and seen and not _used_with_topic(term, seen):
continue # "rozložená kniha" for blockchain: CRZP does not use it, no search continue # "rozložená kniha" for blockchain: CRZP does not use it, no search
results.append(fetch.search(text, links=links, depth=depth, **filters(query))) results.append(read(text))
query.searched.append(text) query.searched.append(text)
seen += results[-1]["rows"] seen += results[-1]["rows"]
if term: if term:
@ -255,6 +338,12 @@ def find(query: CrzpQuery, *, links: bool = True) -> tuple[dict | None, list[dic
continue continue
found = _merge(results) found = _merge(results)
rows = prefer_exact([r for r in found["rows"] if keep(r, query)], query) rows = prefer_exact([r for r in found["rows"] if keep(r, query)], query)
if by_facets(query) and not found["complete"]:
if len(rows) == len(found["rows"]):
found["facet_only"] = True # CRZP itself counted it; every read row meets the conditions
else: # some found works are elsewhere (e.g. a school name in a title): count the rows themselves
found = _merge([fetch.search_deep(r["query"], links=False, **filters(query)) for r in found["parts"]])
rows = prefer_exact([r for r in found["rows"] if keep(r, query)], query)
last = (found, rows) last = (found, rows)
if rows and unverified and not verified_rows: if rows and unverified and not verified_rows:
query.unverified_only = unverified # the topic itself found nothing: say so in the answer query.unverified_only = unverified # the topic itself found nothing: say so in the answer
@ -326,8 +415,8 @@ def work_block(row: dict, detail: dict | None = None) -> str:
def work_line(row: dict) -> str: def work_line(row: dict) -> str:
return (f"- **{row['title']}** ({_kind(row)}, {row.get('year') or '?'}, {school(row)}) — " return (f"- {_names(row.get('authors', []))} — „{row['title']}“ ({_kind(row)}, {row.get('year') or '?'}, "
f"{_names(row.get('authors', []))} · {_link(row)}") f"{school(row)}) · {_link(row)}")
def describe(query: CrzpQuery) -> str: def describe(query: CrzpQuery) -> str:
@ -354,10 +443,13 @@ def describe(query: CrzpQuery) -> str:
# ------------------------------------------------------------------ model steps # ------------------------------------------------------------------ model steps
SUMMARY_PROMPT = ("Na základe abstraktu a kľúčových slov záverečnej práce napíš po slovensky 2 až 3 vety: o čom práca " SUMMARY_PROMPT = ("Si odborník, ktorý stručne vysvetľuje záverečné práce. Z abstraktu a kľúčových slov napíš po "
"je a čo v nej autor urobil. Môžeš preformulovať, ale použi iba informácie z abstraktu; nepridávaj " "slovensky 2 až 3 vlastné vety: o čom práca je, čo v nej autor urobil a s akými nástrojmi alebo "
"výsledky, nástroje ani hodnotenia, ktoré v ňom nie sú. Bez úvodu a bez nadpisu.") "metódami. Píš vlastnými slovami a súvislo, neopisuj vety abstraktu doslova a nezačínaj slovami "
CHECK_PROMPT = ("Over, či je každé tvrdenie zhrnutia podložené abstraktom (preformulovanie je v poriadku, nový fakt nie). " "„Táto práca“ ani „Abstrakt“. Použi iba informácie z abstraktu a kľúčových slov; nepridávaj "
"výsledky, čísla, nástroje ani hodnotenia, ktoré v nich nie sú. Bez nadpisu a bez odrážok.")
CHECK_PROMPT = ("Over, či je každé tvrdenie zhrnutia podložené abstraktom alebo kľúčovými slovami (preformulovanie "
"a zovšeobecnenie je v poriadku, nový fakt, číslo alebo nástroj nie). "
"Odpovedz iba JSON {\"supported\": true} alebo {\"supported\": false}.") "Odpovedz iba JSON {\"supported\": true} alebo {\"supported\": false}.")
@ -373,17 +465,30 @@ def _ask(system: str, payload: dict, complete=None, max_tokens: int = 400) -> st
def summary(detail: dict, complete=None) -> tuple[str, str]: def summary(detail: dict, complete=None) -> tuple[str, str]:
"""("Zhrnutie", text) after a passed check, else ("Abstrakt", abstract).""" """("Zhrnutie", text): the model's own 2-3 sentences after a passed check (second try with
the check's verdict), else a sentence built from the CRZP keywords — never the copied abstract."""
abstract = detail.get("abstract") or detail.get("abstract_secondary") or "" abstract = detail.get("abstract") or detail.get("abstract_secondary") or ""
keywords = detail.get("keywords", [])
if not abstract: if not abstract:
return "Abstrakt", "CRZP pri tejto práci abstrakt neuvádza." return "Zhrnutie", _keyword_sentence(detail) if keywords else "CRZP pri tejto práci abstrakt neuvádza."
text = _ask(SUMMARY_PROMPT, {"title": detail["title"], "abstract": abstract, "keywords": detail.get("keywords", [])}, payload = {"title": detail["title"], "abstract": abstract, "keywords": keywords}
complete) for attempt in range(2):
if text and len(text) < 1200: text = _ask(SUMMARY_PROMPT if attempt == 0 else SUMMARY_PROMPT + " Drž sa výlučne faktov z abstraktu.",
verdict = _ask(CHECK_PROMPT, {"abstract": abstract, "summary": text}, complete, max_tokens=50) payload, complete)
if not text or len(text) > 1200:
continue
verdict = _ask(CHECK_PROMPT, {"abstract": abstract, "keywords": keywords, "summary": text}, complete, max_tokens=50)
if verdict and re.search(r'"supported"\s*:\s*true', verdict): if verdict and re.search(r'"supported"\s*:\s*true', verdict):
return "Zhrnutie podľa abstraktu", text return "Zhrnutie", text
return "Abstrakt", abstract if len(abstract) <= 1500 else abstract[:1500].rsplit(" ", 1)[0] + " …" return "Zhrnutie", _keyword_sentence(detail)
def _keyword_sentence(detail: dict) -> str:
keywords = [k for k in detail.get("keywords", []) if len(k) < 40][:6]
kind = WORK_TYPE_LABELS.get(detail.get("work_type"), "práca").capitalize()
if not keywords:
return f"{kind} sa podľa CRZP zaoberá témou „{detail['title']}“."
return f"{kind} „{detail['title']}“ sa zaoberá témami: {', '.join(keywords)}."
# ------------------------------------------------------------------ answers by intent # ------------------------------------------------------------------ answers by intent
@ -391,6 +496,10 @@ def summary(detail: dict, complete=None) -> tuple[str, str]:
def _list(question, query, found, rows) -> tuple[str, list[dict]]: def _list(question, query, found, rows) -> tuple[str, list[dict]]:
rows = ranked(question, query, rows) rows = ranked(question, query, rows)
header = f"V CRZP som našiel {len(rows)} {'prácu' if len(rows) == 1 else 'práce' if len(rows) < 5 else 'prác'} ({describe(query)}):" header = f"V CRZP som našiel {len(rows)} {'prácu' if len(rows) == 1 else 'práce' if len(rows) < 5 else 'prác'} ({describe(query)}):"
if not found["complete"] and found["total"] > len(found["rows"]):
newest = not query.topics and not query.title and not query.people
header = (f"V CRZP je {found['total']} výsledkov hľadania ({describe(query)}); prehľadaných bolo "
f"{len(found['rows'])} {'najnovších' if newest else 'najrelevantnejších'}, z nich zodpovedá {len(rows)}:")
if len(rows) <= 5: if len(rows) <= 5:
return header + "\n\n" + "\n\n".join(work_block(r) for r in rows), rows return header + "\n\n" + "\n\n".join(work_block(r) for r in rows), rows
shown = rows[:LIST_LIMIT] shown = rows[:LIST_LIMIT]
@ -400,28 +509,35 @@ def _list(question, query, found, rows) -> tuple[str, list[dict]]:
return text, shown return text, shown
TYPE_FACET = {"bakalárska": "bakalárska práca", "diplomová": "diplomová práca", "dizertačná": "dizertačná práca",
"habilitačná": "habilitačná práca", "rigorózna": "rigorózna práca"}
def _facet_type(label: str) -> str:
return next((v for k, v in TYPE_FACET.items() if k in label.lower()), label)
def _count(question, query, found, rows) -> tuple[str, list[dict]]: def _count(question, query, found, rows) -> tuple[str, list[dict]]:
count, note = len(rows), "" count, note = len(rows), ""
person = query.people[0] if query.people else None person = query.people[0] if query.people else None
if not found["complete"]:
facet = FACET.get(person.role) if person else None
values = [f["count"] for f in found["facets"].get(facet, []) if name_matches(person.name, f["label"])] if facet else []
# The facet counts the whole CRZP result, which already respects the year and type filters.
school_only = (query.school and not query.people and not query.topics and not query.title
and found["texts"] == (UNITS[query.unit]["search"] if query.unit else [SCHOOLS[query.school][0]]))
if values and len(found["texts"]) == 1 and not (query.school or query.unit or len(query.people) > 1):
count, note = sum(values), " (súhrnný počet z CRZP)"
elif school_only and len(rows) == len(found["rows"]):
count = found["total"]
several = len(found["texts"]) > 1
note = (f" (súhrn CRZP pre hľadanie {'pracoviska (kód a starší zápis názvom)' if several else 'kódu pracoviska' if query.unit else 'názvu školy'} "
f"s filtrom roku a typu práce; všetkých {len(rows)} prehľadaných záznamov bolo z tohto pracoviska)")
else:
note = f" (aspoň; prehľadaných {len(found['rows'])} z {found['total']} výsledkov hľadania)"
types = Counter(_kind(r) for r in rows) types = Counter(_kind(r) for r in rows)
years = Counter(r.get("year") for r in rows if r.get("year")) years = Counter(r.get("year") for r in rows if r.get("year"))
if found.get("facet_only"):
# CRZP's own total and full facets (texts of one workplace are disjoint: code vs. old written name)
count, types, years = found["total"], Counter(), Counter()
for part in found["parts"]:
years.update(fetch.year_counts(part))
types.update({_facet_type(f["label"]): f["count"] for f in part["facets"].get("Typ práce", [])})
note = " (súhrnný počet z CRZP)"
elif not found["complete"]:
facet = FACET.get(person.role) if person else None
values = [f["count"] for f in found["facets"].get(facet, []) if name_matches(person.name, f["label"])] if facet else []
if values and len(found["texts"]) == 1 and not (query.school or query.unit or len(query.people) > 1):
count, note = sum(values), " (súhrnný počet z CRZP)"
else:
note = f" (aspoň; prehľadaných {len(found['rows'])} z {found['total']} výsledkov hľadania)"
text = f"Počet prác v CRZP ({describe(query)}): **{count}**{note}." text = f"Počet prác v CRZP ({describe(query)}): **{count}**{note}."
if rows: if count:
text += "\n\n- Podľa typu: " + ", ".join(f"{k} {v}" for k, v in types.most_common()) text += "\n\n- Podľa typu: " + ", ".join(f"{k} {v}" for k, v in types.most_common())
text += "\n- Podľa roku: " + ", ".join(f"{y}: {n}" for y, n in sorted(years.items(), reverse=True)) text += "\n- Podľa roku: " + ", ".join(f"{y}: {n}" for y, n in sorted(years.items(), reverse=True))
return text, rows return text, rows
@ -443,11 +559,15 @@ def _about(question, query, found, rows, complete=None) -> tuple[str, list[dict]
summaries = list(pool.map(lambda d: summary(d, complete) if d else None, details)) summaries = list(pool.map(lambda d: summary(d, complete) if d else None, details))
blocks = [] blocks = []
for row, detail, done in zip(rows, details, summaries): for row, detail, done in zip(rows, details, summaries):
head = (f"**{row['title']}** — {_kind(row)}, {row.get('year') or 'rok neuvedený'}, {university(row)}; "
f"autor {_names(row.get('authors', []))}")
if re.search(r"oponent|ved[uú]c|[sš]kolite", question.lower()): # "Kto bol oponentom … a o čom je?"
head += f"; vedúci {_names(row.get('supervisors', []))}; oponent {_names(row.get('opponents', []))}"
if detail is None: if detail is None:
blocks.append(work_block(row) + "\n\nDetail záznamu (abstrakt) nie je dostupný bez trvalého odkazu.") blocks.append(head + "\n\nCRZP k tejto práci neposkytuje trvalý odkaz, preto abstrakt nemám.")
continue continue
label, text = done _, text = done
blocks.append(work_block(row, detail) + f"\n\n{label}: {text}") blocks.append(head + f"\n\n{text}\n\nZdroj: {_link(row)}")
more = len(candidates) > len(rows) more = len(candidates) > len(rows)
tail = "\n\nZodpovedá viac prác; zobrazené sú najrelevantnejšie. Spresni typ práce alebo rok." if more else "" tail = "\n\nZodpovedá viac prác; zobrazené sú najrelevantnejšie. Spresni typ práce alebo rok." if more else ""
return "\n\n".join(blocks) + tail, rows return "\n\n".join(blocks) + tail, rows
@ -478,6 +598,15 @@ def _relation(question, query, found, rows) -> tuple[str, list[dict]]:
plural = lambda c: f"{c} {'práca' if c == 1 else 'práce' if c < 5 else 'prác'}" plural = lambda c: f"{c} {'práca' if c == 1 else 'práce' if c < 5 else 'prác'}"
if not query.people: # "Kto viedol práce o X?": the target role over the found works if not query.people: # "Kto viedol práce o X?": the target role over the found works
counts = Counter(n for r in rows for n in r.get(ROLE_FIELD[target], [])) counts = Counter(n for r in rows for n in r.get(ROLE_FIELD[target], []))
if (found.get("facet_only") and target == "supervisor" and not found["complete"]
and all(part["facets"].get("Školiteľ") for part in found["parts"])): # CRZP's full supervisor facet
counts = Counter()
for part in found["parts"]:
counts.update({f["label"]: f["count"] for f in part["facets"].get("Školiteľ", [])})
total = found["total"]
lines = [f"- {display_name(n)} ({plural(c)})" for n, c in sorted(counts.items(), key=lambda i: (-i[1], i[0]))[:15]]
return (f"{ROLE_PLURAL[target]} prác ({describe(query)}; {total} prác v CRZP, súhrn CRZP):\n\n" +
("\n".join(lines) or "- CRZP ich neuvádza")), ranked(question, query, rows)[:10]
lines = [f"- {display_name(n)} ({plural(c)})" for n, c in sorted(counts.items(), key=lambda i: (-i[1], i[0]))[:15]] lines = [f"- {display_name(n)} ({plural(c)})" for n, c in sorted(counts.items(), key=lambda i: (-i[1], i[0]))[:15]]
return (f"{ROLE_PLURAL[target]} prác ({describe(query)}; {len(rows)} prác v CRZP):\n\n" + return (f"{ROLE_PLURAL[target]} prác ({describe(query)}; {len(rows)} prác v CRZP):\n\n" +
("\n".join(lines) or "- CRZP ich neuvádza")), ranked(question, query, rows)[:10] ("\n".join(lines) or "- CRZP ich neuvádza")), ranked(question, query, rows)[:10]
@ -565,8 +694,8 @@ def _similar(question, query, found, rows) -> tuple[str, list[dict]]:
if not pairs: if not pairs:
return (f"K práci **{work['title']}** som medzi prácami s jej kľúčovými slovami nenašiel inú so spoločnými " return (f"K práci **{work['title']}** som medzi prácami s jej kľúčovými slovami nenašiel inú so spoločnými "
f"kľúčovými slovami. · {_link(work)}"), [work] f"kľúčovými slovami. · {_link(work)}"), [work]
lines = [f"- **{c['title']}** ({_kind(c)}, {c.get('year') or '?'}, {school(c)}) — spoločné: " lines = [f"- {_names(c.get('authors', []))} — {university(c)}: „{c['title']}“ ({_kind(c)}, {c.get('year') or '?'}); "
f"{', '.join(shared)} · {_link(c)}" for c, shared in pairs] f"spoločné: {', '.join(shared)} · {_link(c)}" for c, shared in pairs]
text = (f"Práce so spoločnými kľúčovými slovami CRZP ako **{work['title']}** ({_link(work)}); porovnané s prácami " text = (f"Práce so spoločnými kľúčovými slovami CRZP ako **{work['title']}** ({_link(work)}); porovnané s prácami "
f"nájdenými podľa kľúčových slov „{'“, „'.join(searched)}“:\n\n" + "\n".join(lines)) f"nájdenými podľa kľúčových slov „{'“, „'.join(searched)}“:\n\n" + "\n".join(lines))
return text, [work] + [c for c, _ in pairs] return text, [work] + [c for c, _ in pairs]
@ -663,7 +792,7 @@ def _answer_query(question: str, query: CrzpQuery, complete=None) -> dict:
return _result(question, "no_answer", "Na vyhľadanie v CRZP uveď meno študenta, vedúceho alebo oponenta, " return _result(question, "no_answer", "Na vyhľadanie v CRZP uveď meno študenta, vedúceho alebo oponenta, "
"názov práce alebo tému.", reason="missing_search_terms") "názov práce alebo tému.", reason="missing_search_terms")
try: try:
found, rows = find(query, links=query.intent != "count") found, rows = find(query)
if found is None: if found is None:
return _result(question, "no_answer", "Otázke nezodpovedá žiadne vyhľadávanie v CRZP.", return _result(question, "no_answer", "Otázke nezodpovedá žiadne vyhľadávanie v CRZP.",
reason="missing_search_terms") reason="missing_search_terms")
@ -703,9 +832,9 @@ def _answer_query(question: str, query: CrzpQuery, complete=None) -> dict:
terms = _without_suffix(query, found["texts"]) terms = _without_suffix(query, found["texts"])
if len(terms) > 1 and query.topics: if len(terms) > 1 and query.topics:
body += "\n\nHľadané výrazy v CRZP: „" + "“, „".join(terms) + "“ (vrátane príbuzných výrazov témy)." body += "\n\nHľadané výrazy v CRZP: „" + "“, „".join(terms) + "“ (vrátane príbuzných výrazov témy)."
truncated = not found["complete"] truncated = not found["complete"] and not found.get("facet_only") # a CRZP total needs no read rows
reasons = ["crzp_result_limit"] if truncated else [] reasons = ["crzp_result_limit"] if truncated else []
for text, total, read in found["incomplete"]: for text, total, read in (found["incomplete"] if truncated else []):
body += (f"\n\nPoznámka: hľadanie „{text}“ vrátilo v CRZP {total} záznamov; prehľadaných bolo prvých {read}.") body += (f"\n\nPoznámka: hľadanie „{text}“ vrátilo v CRZP {total} záznamov; prehľadaných bolo prvých {read}.")
body += "\n\nZdroj: Centrálny register záverečných prác (CRZP), metadáta záznamov." body += "\n\nZdroj: Centrálny register záverečných prác (CRZP), metadáta záznamov."
result = _result(question, "ok", body, rows=used, complete=not truncated, truncated=truncated, reasons=reasons, result = _result(question, "ok", body, rows=used, complete=not truncated, truncated=truncated, reasons=reasons,

3
tools/crzp/answering/context.py
View File

@ -26,7 +26,7 @@ SCHEMA = "crzp-1"
LIFETIME = 24 * 60 * 60 LIFETIME = 24 * 60 * 60
MAX_TOKEN = 16000 MAX_TOKEN = 16000
MAX_SIDS = 30 MAX_SIDS = 30
CARRIED = {"intent", "people", "title", "topics", "surface", "phrases", "school_name", "years", "work_types", "school", "unit", CARRIED = {"intent", "people", "title", "topics", "surface", "phrases", "school_name", "works", "years", "work_types", "school", "unit",
"alternatives", "target_role", "from_model", "sids"} "alternatives", "target_role", "from_model", "sids"}
CONTINUES = re.compile(r"^(?:a|aj|a co|co tak|a ako|a kto|a kolko|a ktore|a v|a z|a pre|a len|iba|len|teraz)\b") CONTINUES = re.compile(r"^(?:a|aj|a co|co tak|a ako|a kto|a kolko|a ktore|a v|a z|a pre|a len|iba|len|teraz)\b")
@ -144,6 +144,7 @@ def follow_up(question: str, new: CrzpQuery, previous: dict) -> tuple[CrzpQuery,
if chosen: if chosen:
merged.people[0].name = merged.people[0].typed = chosen["name"] merged.people[0].name = merged.people[0].typed = chosen["name"]
merged.school_name = chosen["school"] merged.school_name = chosen["school"]
merged.works = list(chosen.get("works") or [])
merged.from_model = True # the exact CRZP name: no other name forms merged.from_model = True # the exact CRZP name: no other name forms
return merged, None return merged, None
if ordinal is not None and not own_subject: if ordinal is not None and not own_subject:

60
tools/crzp/answering/lexicon.py Normal file
View File

@ -0,0 +1,60 @@
"""CRZP's own vocabulary for questions answered without the model.
CRZP matches exact words, so an inflected topic ("rozpoznávaním emócií") must
be turned into the form the records use. The tags (keywords) of every record
read before are counted; an inflected phrase is mapped to the most frequent
tag with the same number of words whose words match it one by one up to
inflection ("rozpoznávanie emócií", "neurónové siete"). Nothing is invented:
only phrases CRZP records really carry are used.
"""
from __future__ import annotations
import threading
import time
from collections import Counter
from tools.crzp.source.parse import fold
REFRESH_SECONDS = 3600
_lock = threading.Lock()
_state = {"built": 0.0, "phrases": {}}
def _stem_equal(a: str, b: str) -> bool:
n = max(3, min(len(a), len(b)) - 2)
return len(a) >= 3 and len(b) >= 3 and a[:n] == b[:n] and abs(len(a) - len(b)) <= 4
def _build() -> dict:
from tools.crzp.fetch import store
counts, spelled = Counter(), {}
for row in store().cached_rows():
for tag in row.get("keywords", []):
key = " ".join(fold(tag).split())
if 1 <= len(key.split()) <= 4 and len(key) <= 60:
counts[key] += 1
spelled.setdefault(key, tag.strip())
by_words: dict = {}
for key, count in counts.items():
by_words.setdefault((len(key.split()), key.split()[0][:3]), []).append((count, key))
return {"by_words": by_words, "spelled": spelled}
def vocabulary() -> dict:
with _lock:
if time.monotonic() - _state["built"] > REFRESH_SECONDS:
_state["phrases"] = _build()
_state["built"] = time.monotonic()
return _state["phrases"]
def tag_for(words: list[str]) -> str | None:
"""The most frequent CRZP tag matching the inflected words one by one, or None."""
folded = [fold(w) for w in words if w]
if not folded or len(folded) > 4:
return None
table = vocabulary()
candidates = table.get("by_words", {}).get((len(folded), folded[0][:3]), [])
best = max(((count, key) for count, key in candidates
if all(_stem_equal(a, b) for a, b in zip(folded, key.split()))), default=None)
return table["spelled"][best[1]] if best else None

23
tools/crzp/answering/understand.py
View File

@ -105,6 +105,7 @@ class CrzpQuery(BaseModel):
relaxed_author: bool = False # asked as author, found only in another role relaxed_author: bool = False # asked as author, found only in another role
sids: list[str] = Field(default_factory=list, max_length=30) # follow-up: only these CRZP records sids: list[str] = Field(default_factory=list, max_length=30) # follow-up: only these CRZP records
school_name: str | None = None # follow-up: the chosen namesake's university (answer.university) school_name: str | None = None # follow-up: the chosen namesake's university (answer.university)
works: list[str] = Field(default_factory=list, max_length=30) # follow-up: the chosen namesake's works
searched: list[str] = Field(default_factory=list) # texts actually sent to CRZP (set by answer.find) searched: list[str] = Field(default_factory=list) # texts actually sent to CRZP (set by answer.find)
unverified_only: list[str] = Field(default_factory=list) # found only under unverified proposed terms unverified_only: list[str] = Field(default_factory=list) # found only under unverified proposed terms
@ -173,6 +174,8 @@ def asks_for_works(low: str) -> bool:
def _intent(low: str) -> Intent: def _intent(low: str) -> Intent:
if OUT_OF_SCOPE.search(fold(low)): if OUT_OF_SCOPE.search(fold(low)):
return "refused" return "refused"
if re.search(r"^\W*kto\b[^?]*\b(?:viedol|viedla|viedli|oponoval|oponovala|oponovali|[sš]kolil)\w*\b[^?]*\bpr[aá]c", low):
return "relation" # "Kto viedol najviac prác o X?": the supervisors, not a count
if re.search(r"\bkoľk\w*|\bkolk\w*|\bpočet\b|\bpocet\b|\bhow many\b", low): if re.search(r"\bkoľk\w*|\bkolk\w*|\bpočet\b|\bpocet\b|\bhow many\b", low):
return "count" return "count"
if re.search(r"kľúčov|klucov|\btag|keyword", low): if re.search(r"kľúčov|klucov|\btag|keyword", low):
@ -264,6 +267,18 @@ def _rule_people_topics(question: str, query: CrzpQuery) -> None:
query.topics = list(dict.fromkeys(topics))[:4] query.topics = list(dict.fromkeys(topics))[:4]
query.surface = [" ".join(surface[:2])] if surface else [] query.surface = [" ".join(surface[:2])] if surface else []
query.phrases = topic_phrases(runs[0])[:3] if runs else [] query.phrases = topic_phrases(runs[0])[:3] if runs else []
if runs: # the tag CRZP records really use for these words comes first
from tools.crzp.answering.lexicon import tag_for
try:
tag = tag_for([token for token, _ in runs[0]])
except Exception:
tag = None
if tag:
query.phrases = [tag, *query.phrases]
unique = {}
for phrase in query.phrases: # CRZP ignores diacritics: "rozpoznavanie" is "rozpoznávanie"
unique.setdefault(" ".join(fold(phrase).split()), phrase)
query.phrases = list(unique.values())[:3]
PLURAL_NOUN = [("iach", "ie"), ("ach", "e"), ("och", "y"), ("ami", "e"), ("mi", "y"), ("ov", "y"), ("í", "e")] PLURAL_NOUN = [("iach", "ie"), ("ach", "e"), ("och", "y"), ("ami", "e"), ("mi", "y"), ("ov", "y"), ("í", "e")]
@ -351,7 +366,7 @@ def understand(question: str, *, complete=None, use_model: bool = True) -> CrzpQ
query.topics = [t.strip() for t in proposed.get("topics") or [] query.topics = [t.strip() for t in proposed.get("topics") or []
if isinstance(t, str) and t.strip() and in_question(t, question) and not work_words_only(t) if isinstance(t, str) and t.strip() and in_question(t, question) and not work_words_only(t)
and not names_a_school(t) and not any(in_question(t, p.name) for p in query.people)][:4] and not names_a_school(t) and not any(in_question(t, p.name) for p in query.people)][:4]
if (query.intent == "list" and proposed.get("intent") in ("count", "about", "keywords", "relation", "similar", "schools") if (query.intent in ("list",) and proposed.get("intent") in ("count", "about", "keywords", "relation", "similar", "schools")
and not (proposed["intent"] == "about" and asks_for_works(question.lower()))): and not (proposed["intent"] == "about" and asks_for_works(question.lower()))):
query.intent = proposed["intent"] # rules found no cue (e.g. another language) query.intent = proposed["intent"] # rules found no cue (e.g. another language)
title = proposed.get("title") title = proposed.get("title")
@ -366,6 +381,12 @@ def understand(question: str, *, complete=None, use_model: bool = True) -> CrzpQ
offered = [] offered = []
if query.topics and not query.people and not query.title: if query.topics and not query.people and not query.title:
query.alternatives = alternatives(query.topic_key(), offered) query.alternatives = alternatives(query.topic_key(), offered)
who = re.match(r"^\W*kto\b[^?]*?\b(viedol|viedla|viedli|oponoval|oponovala|oponovali|[sš]kolil)", question.lower())
if who and query.intent == "relation": # "Kto oponoval práce Hládka?": the opponents of Hládek's works
query.target_role = "opponent" if who.group(1).startswith("opon") else "supervisor"
for person in query.people:
if person.role == query.target_role:
person.role = "any"
if query.target_role is None and query.intent == "relation": if query.target_role is None and query.intent == "relation":
asked = [r for r, cue in ROLE_CUES.items() if re.search(cue, question.lower()) asked = [r for r, cue in ROLE_CUES.items() if re.search(cue, question.lower())
and all(p.role != r for p in query.people)] and all(p.role != r for p in query.people)]

38
tools/crzp/answering/units.py
View File

@ -13,15 +13,20 @@ Older records (about until 2015) write the department by name ("1040 Katedra
elektroniky a multimediálnych telekomunikácií") instead of 104005; CRZP indexes elektroniky a multimediálnych telekomunikácií") instead of 104005; CRZP indexes
that text too, so a workplace is searched by every form in "search" and the that text too, so a workplace is searched by every form in "search" and the
name is matched as written. name is matched as written.
Other codes (e.g. 104009, 104004) are shown as numbers, never guessed. Other TUKE department codes are named only when workplaces.json holds a name
learned from CRZP records with enough votes (see workplaces.py); such a
workplace can be asked by its acronym ("na KPI") or name and is searched by
its code and its old written name. Codes without evidence stay numbers
("pracovisko 104009"), never guessed.
""" """
from __future__ import annotations from __future__ import annotations
import re import re
from tools.crzp.answering import workplaces
from tools.crzp.source.parse import fold from tools.crzp.source.parse import fold
UNITS = { VERIFIED = {
"kemt": {"school": "tuke", "code": "104005", "label": "KEMT FEI", "kemt": {"school": "tuke", "code": "104005", "label": "KEMT FEI",
"text": "katedra elektroniky a multimedialnych telekomunikacii", "text": "katedra elektroniky a multimedialnych telekomunikacii",
"search": ["104005", "Katedra elektroniky a multimediálnych telekomunikácií"]}, "search": ["104005", "Katedra elektroniky a multimediálnych telekomunikácií"]},
@ -34,10 +39,28 @@ CUES = {
} }
def _learned_units() -> dict:
"""Departments named in workplaces.json, keyed by their acronym ("kpi")."""
units = {}
for short, (code, entry) in workplaces.by_acronym(workplaces.TUKE).items():
if short in VERIFIED or code in {u["code"] for u in VERIFIED.values()}:
continue
units[short] = {"school": "tuke", "code": code, "label": f"{entry['short']} FEI" if code.startswith("1040")
else entry["short"], "text": fold(entry["name"]), "search": [code, entry["name"]],
"learned": True}
words = [w for w in fold(entry["name"]).split() if w not in workplaces.SMALL_WORDS]
CUES[short] = r"\b" + re.escape(short) + r"\b|" + r"\w*\s+(?:\w+\s+)?".join(re.escape(w[:5]) for w in words)
return units
UNITS = {**VERIFIED, **_learned_units()}
ORDER = ["kemt", *[k for k in UNITS if UNITS[k].get("learned")], "fei"] # the most specific unit first
def unit_in_question(question: str) -> str | None: def unit_in_question(question: str) -> str | None:
"""The most specific verified unit the question names (KEMT before FEI).""" """The most specific known unit the question names (a department before FEI)."""
low = question.lower() low = fold(question)
return next((key for key in ("kemt", "fei") if re.search(CUES[key], low)), None) return next((key for key in ORDER if re.search(fold(CUES[key]) if key in ("kemt", "fei") else CUES[key], low)), None)
def _tuke(row: dict) -> bool: def _tuke(row: dict) -> bool:
@ -64,6 +87,7 @@ def unit_label(row: dict) -> str:
part = part.strip(" ,") part = part.strip(" ,")
if not part: if not part:
continue continue
known = next((u["label"].split()[0] for u in UNITS.values() if u["code"] == part), None) known = next((u["label"].split()[0] for u in VERIFIED.values() if u["code"] == part), None)
names.append(known or (f"pracovisko {part}" if part.isdigit() else part)) entry = workplaces.name_of(workplaces.TUKE, part) if part.isdigit() and not known else None
names.append(known or (f"{entry['name']} ({part})" if entry else f"pracovisko {part}" if part.isdigit() else part))
return ", ".join(names) return ", ".join(names)

88
tools/crzp/answering/wiki_bridge.py Normal file
View File

@ -0,0 +1,88 @@
"""ZP Wiki -> CRZP: a thesis fact the wiki profile lacks, taken from CRZP.
A student's ZP Wiki profile may be incomplete (no opponent, no keywords, no
abstract, no final title). When /rag or /graphrag answers "no_answer" for a
person the wiki knows, the same question is asked in CRZP for exactly that
person (the canonical wiki name) at Technická univerzita v Košiciach, the school
of ZP Wiki. The answer keeps the wiki's own message first and marks the CRZP
part with its source, so it is never presented as a wiki fact.
Not used for what CRZP does not hold (meetings, progress, tasks, assignments,
results beyond the abstract, repositories) and never for an unknown or
ambiguous person, a refused question or another failure. CRZP_WIKI_FALLBACK=0
turns it off.
"""
from __future__ import annotations
import os
import re
from tools.answers.context_policy import normalized
from tools.answers.delivery_contract import fingerprint
# What only the wiki records: CRZP is not asked.
WIKI_ONLY = re.compile(r"\b(?:vysled\w*|stretnut\w*|pokrok\w*|progres\w*|uloh\w*|zadan\w*|plan\w*|termin\w*|"
r"tyzd\w*|repozitar\w*|git\w*|kod\w*|stav\w*|poznam\w*|konzult\w*|zaciat\w*|studi\w*|"
r"obhaj\w*|obhaj\w*|ukonc\w*|dokonc\w*|skoncil\w*|uspes\w*|statnic\w*|hodnot\w*|znamk\w*)\b")
# A CRZP record means a submitted work, not a defended or graded one: completion stays a wiki question.
INTENTS = ("list", "about", "keywords", "relation", "count")
def enabled() -> bool:
return os.getenv("CRZP_WIKI_FALLBACK", "1") != "0"
def wiki_person(question: str, db_file=None) -> str | None:
"""The one ZP Wiki person the question names (canonical name), else None."""
match = wiki_match(question, db_file)
return match["name"] if match else None
def wiki_match(question: str, db_file=None) -> dict | None:
"""{"name", "student"}: the one ZP Wiki person the question names; student = has a profile page."""
from tools.answers.question_plan import planning_catalog
from tools.rag.common import DB_FILE
from tools.retrieval.person_names import resolve_person_names
try:
catalog = planning_catalog(db_file or DB_FILE)
except (OSError, ValueError, RuntimeError):
return None
resolution = resolve_person_names(question, catalog or [])
matches = resolution.get("matches") or []
if resolution.get("status") != "ok" or len({m["name"] for m in matches}) != 1:
return None
source = matches[0].get("source_document") or ""
return {"name": matches[0]["name"], "student": source.startswith(("pages/students/", "pages/interns/"))}
def supplement(question: str, wiki_result: dict, engine: str, *, db_file=None, complete=None) -> dict | None:
"""The wiki result extended by the CRZP answer for the same person, or None."""
if not enabled() or wiki_result.get("status") != "no_answer" or WIKI_ONLY.search(normalized(question)):
return None
person = wiki_person(question, db_file)
if not person:
return None
from tools.crzp.answering import answer as crzp
from tools.crzp.answering.understand import Person, understand
query = understand(question, complete=complete)
if query.intent not in INTENTS:
return None
role = query.people[0].role if query.people else "any"
query.people = [Person(name=person, role=role, typed=person)]
query.from_model = True # the canonical wiki name: no other name forms
query.school, query.school_name = "tuke", None
query.title = query.title if query.title and person not in query.title else None
result = crzp._answer_query(question, query, complete)
if result.get("status") != "ok":
return None
wiki_text = (wiki_result.get("answer") or "").strip()
text = ((wiki_text + "\n\n" if wiki_text else "") + "**Doplnené z CRZP** (Centrálny register záverečných prác, "
f"Technická univerzita v Košiciach; ZP Wiki tento údaj neuvádza):\n\n" + result["answer"])
truncated = bool(result.get("truncated"))
return {**{k: v for k, v in wiki_result.items() if k not in ("answer", "reason", "delivery_check")},
"status": "ok", "reason": None, "accepted_for_delivery": True, "answer": text,
"source_urls": list(dict.fromkeys([*(wiki_result.get("source_urls") or []), *result["source_urls"]])),
"claims": [], "complete": not truncated, "truncated": truncated,
"truncation_reasons": result.get("truncation_reasons", []), "supplement": "crzp",
"delivery_check": {"layer": "delivery", "code": "partial" if truncated else "complete",
"request_fingerprint": fingerprint(question, engine)}}

144
tools/crzp/answering/workplaces.json Normal file
View File

@ -0,0 +1,144 @@
{
"Technická univerzita v Košiciach": {
"101201": {
"lead": 6,
"name": "Ústav podnikania a manažmentu",
"short": "ÚPM",
"voters": 6,
"votes": 6
},
"101401": {
"lead": 4,
"name": "Ústav montánnych vied a ochrany životného prostredia",
"short": "ÚMVOŽP",
"voters": 6,
"votes": 5
},
"101601": {
"lead": 5,
"name": "Ústav logistiky priemyslu a dopravy",
"short": "ÚLPD",
"voters": 5,
"votes": 5
},
"103102": {
"lead": 4,
"name": "Katedra aplikovanej mechaniky a mechatroniky",
"short": "KAMM",
"voters": 4,
"votes": 4
},
"103201": {
"lead": 6,
"name": "Katedra technológií a materiálov",
"short": "KTM",
"voters": 6,
"votes": 6
},
"103203": {
"lead": 6,
"name": "Katedra manažmentu a ekonomiky",
"short": "KME",
"voters": 6,
"votes": 6
},
"103402": {
"lead": 5,
"name": "Katedra environmentalistiky a riadenia procesov",
"short": "KERP",
"voters": 5,
"votes": 5
},
"104004": {
"lead": 6,
"name": "Katedra kybernetiky a umelej inteligencie",
"short": "KKUI",
"voters": 6,
"votes": 6
},
"104007": {
"lead": 5,
"name": "Katedra elektroenergetiky",
"short": "KE",
"voters": 5,
"votes": 5
},
"104009": {
"lead": 6,
"name": "Katedra počítačov a informatiky",
"short": "KPI",
"voters": 6,
"votes": 6
},
"105201": {
"lead": 5,
"name": "Ústav budov a prostredia",
"short": "ÚBP",
"voters": 5,
"votes": 5
},
"106003": {
"lead": 4,
"name": "Katedra výrobných technológií",
"short": "KVT",
"voters": 4,
"votes": 4
},
"106005": {
"lead": 2,
"name": "Katedra manažmentu výroby",
"short": "KMV",
"voters": 5,
"votes": 3
},
"107002": {
"lead": 3,
"name": "Katedra bankovníctva a investovania",
"short": "KBI",
"voters": 5,
"votes": 4
},
"107003": {
"lead": 2,
"name": "Katedra ekonomických teórií",
"short": "KET",
"voters": 4,
"votes": 3
},
"108003": {
"lead": 4,
"name": "Katedra architektúry",
"short": "KA",
"voters": 4,
"votes": 4
},
"109003": {
"lead": 2,
"name": "Katedra avioniky",
"short": "KA",
"voters": 4,
"votes": 3
},
"109005": {
"lead": 2,
"name": "Katedra letovej prípravy",
"short": "KLP",
"voters": 4,
"votes": 3
},
"109006": {
"lead": 3,
"name": "Katedra leteckého inžinierstva",
"short": "KLI",
"voters": 5,
"votes": 4
},
"109007": {
"lead": 5,
"name": "Katedra manažmentu leteckej prevádzky",
"short": "KMLP",
"voters": 5,
"votes": 5
}
}
}

180
tools/crzp/answering/workplaces.py Normal file
View File

@ -0,0 +1,180 @@
"""Names of CRZP workplace codes, learned from CRZP records themselves.
CRZP writes newer works as "Technická univerzita v Košiciach 1040 104009" and
does not publish what 104009 means. Older records (about until 2015) write the
department by name ("1040 Katedra počítačov a informatiky"). A supervisor stays
at a department, so the name is learned per code:
1. search the code; the full supervisor facet of that result gives the people
who supervise works with this code;
2. for the most frequent supervisors read their older works (OLD_YEARS) at the
same university and collect the department names written there;
3. each supervisor votes once, for the department named most often in his or
her older works (spellings cut by CRZP, "… umelej inteligenci", count as the
full name);
4. accept a name only when at least MIN_VOTES supervisors agree, they are at
least MIN_SHARE of the voters and the name leads the next one by MARGIN votes;
5. a name that wins for several codes is kept only for the code with the
clearest vote (the others stay numbers); placeholder codes (999999) are skipped.
The result is stored in workplaces.json (committed, reviewable) with its votes,
so a code is named only with evidence; other codes stay "pracovisko 104009".
python -m tools.crzp.answering.workplaces learn # TUKE codes seen in the cache
python -m tools.crzp.answering.workplaces learn 104009 104004
"""
from __future__ import annotations
import argparse
import json
import re
from collections import Counter
from functools import lru_cache
from pathlib import Path
from tools.crzp.source.parse import display_name, fold
FILE = Path(__file__).with_name("workplaces.json")
TUKE = "Technická univerzita v Košiciach"
OLD_YEARS = (2005, 2015)
SUPERVISORS = 6
MIN_VOTES = 3
MIN_SHARE = 0.6
MARGIN = 2
PLACEHOLDERS = {"999999", "000000"}
SMALL_WORDS = {"a", "v", "vo", "pre", "na", "so", "s", "z", "zo", "k", "ku", "o", "a."}
@lru_cache(maxsize=1)
def learned() -> dict:
"""{university: {code: {"name", "short", "votes", "voters"}}} from workplaces.json."""
try:
return json.loads(FILE.read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
def name_of(university: str, code: str) -> dict | None:
return learned().get(university, {}).get(code)
def acronym(name: str) -> str:
""""Katedra počítačov a informatiky" -> "KPI"."""
return "".join(w[0].upper() for w in re.findall(r"[^\W\d_]+", name) if w.lower() not in SMALL_WORDS)
def by_acronym(university: str = TUKE) -> dict[str, tuple[str, dict]]:
"""Lower-case acronym -> (code, entry) for acronyms that name exactly one learned workplace."""
seen: dict[str, list] = {}
for code, entry in learned().get(university, {}).items():
seen.setdefault(entry["short"].lower(), []).append((code, entry))
return {k: v[0] for k, v in seen.items() if len(v) == 1 and len(k) >= 3}
CUT_WORDS = {"inteligenci": "inteligencie"} # CRZP cut old names at 40 characters
def _department(unit: str) -> str | None:
"""The written department of an old record: "1040 Katedra počítačov a informatiky"."""
text = re.sub(r"^\s*[\d\s]+", "", unit or "").strip(" ,")
text = re.sub(r"^(Katedra|Ústav)(?=[a-záäčďéíĺľňóôŕšťúýž])", r"\1 ", text) # "Katedraaplikovanej"
words = text.split()
if words and words[-1] in CUT_WORDS:
text = " ".join([*words[:-1], CUT_WORDS[words[-1]]])
return text if re.search(r"(?i)\b(katedr|ústav|ustav|centrum|pracovisk)\w*", text) else None
def codes_in_cache(university: str = TUKE) -> list[str]:
from tools.crzp.answering.answer import university as university_of
from tools.crzp.fetch import store
counts = Counter(code for row in store().cached_rows() if university_of(row) == university
for code in re.findall(r"\b\d{6}\b", row.get("school_unit") or ""))
return [code for code, _ in counts.most_common()]
def learn_code(code: str, university: str = TUKE, *, search=None) -> dict:
"""Votes for the department name of one code (see the module docstring)."""
from tools.crzp import fetch
from tools.crzp.answering.answer import university as university_of
search = search or fetch.search
result = search(code, links=False, depth=20)
supervisors = [display_name(f["label"]) for f in result["facets"].get("Školiteľ", [])][:SUPERVISORS]
votes, voters, spelled = Counter(), 0, {}
for name in supervisors:
old = search(name, years=OLD_YEARS, links=False, depth=200)
surname = fold(name.split()[-1])
own = Counter(_department(r.get("school_unit")) for r in old["rows"] if university_of(r) == university
and any(surname in fold(s) for s in r.get("supervisors", [])))
own.pop(None, None)
merged = Counter()
for department, n in own.items():
key = _canonical(department, spelled)
merged[key] += n
ranked = merged.most_common(2)
if ranked and (len(ranked) == 1 or ranked[0][1] > ranked[1][1]): # one clear department of this person
voters += 1
votes[ranked[0][0]] += 1
if not votes:
return {"code": code, "accepted": False, "voters": voters, "votes": {}}
final = Counter() # a key may have grown to a longer spelling after it got votes
for key, n in votes.items():
final[next((k for k in spelled if k.startswith(key)), key)] += n
votes = final
ranked = votes.most_common(2)
best, count = ranked[0]
second = ranked[1][1] if len(ranked) > 1 else 0
accepted = count >= MIN_VOTES and count / voters >= MIN_SHARE and count - second >= MARGIN
best = spelled[best]
return {"code": code, "accepted": accepted, "name": best, "short": acronym(best),
"votes": {spelled[k]: v for k, v in votes.items()}, "lead": count - second,
"voters": voters, "supervisors": supervisors}
def _canonical(department: str, spelled: dict) -> str:
"""Folded key shared by a name and its cut spellings; `spelled` keeps the longest form."""
folded = " ".join(fold(department).replace("katedraa", "katedra a").split())
for key in list(spelled):
if key.startswith(folded) or folded.startswith(key):
longer = folded if len(folded) > len(key) else key
text = department if len(folded) > len(key) else spelled[key]
if longer != key:
spelled[longer] = text
spelled.pop(key)
return longer
spelled[folded] = department
return folded
def learn(codes: list[str], university: str = TUKE) -> dict:
report = [learn_code(code, university) for code in codes if code not in PLACEHOLDERS]
winners: dict = {}
for outcome in report:
if outcome["accepted"] and outcome["code"] != "104005": # KEMT is verified by hand (units.py)
key = fold(outcome["name"])
if key not in winners or (outcome["lead"], outcome["voters"]) > (winners[key]["lead"], winners[key]["voters"]):
winners[key] = outcome
table = {o["code"]: {"name": o["name"], "short": o["short"], "votes": o["votes"][o["name"]],
"voters": o["voters"], "lead": o["lead"]} for o in winners.values()}
data = {**learned(), university: dict(sorted(table.items()))}
FILE.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
learned.cache_clear()
return {"learned": len(table), "report": report}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("learn")
run.add_argument("codes", nargs="*")
run.add_argument("--limit", type=int, default=25)
args = parser.parse_args()
codes = args.codes or ["104005", *[c for c in codes_in_cache() if c != "104005"][: args.limit]]
result = learn(codes)
for line in result["report"]:
print(json.dumps({k: line.get(k) for k in ("code", "accepted", "name", "voters", "lead", "votes")}, ensure_ascii=False))
print(json.dumps({"learned": result["learned"]}))
return 0
if __name__ == "__main__":
raise SystemExit(main())

84
tools/crzp/fetch.py
View File

@ -19,11 +19,13 @@ import threading
import time import time
from tools.crzp.source.client import CrzpSession, CrzpUnavailable, PERMALINK from tools.crzp.source.client import CrzpSession, CrzpUnavailable, PERMALINK
from tools.crzp.source.parse import fold, parse_detail, parse_results, parse_rows, parse_rss from tools.crzp.source.parse import facet_actions, fold, parse_detail, parse_results, parse_rows, parse_rss
from tools.crzp.storage.cache import CrzpStore from tools.crzp.storage.cache import CrzpStore
LINKED_ROWS = 200 # the RSS export of CRZP is capped at 200 items LINKED_ROWS = 200 # the RSS export of CRZP is capped at 200 items
DEEP_ROWS = int(os.getenv("CRZP_DEEP_ROWS", "1000")) # counts and summaries: up to 10 pages of 100 DEEP_ROWS = int(os.getenv("CRZP_DEEP_ROWS", "3000")) # summaries: read in year ranges, several at once
PARALLEL = int(os.getenv("CRZP_PARALLEL", "3")) # year ranges read at the same time (shared throttle)
EXPANDED = ("Rok odovzdania", "Školiteľ") # facets read in full when a result exceeds one page
MAX_ROWS = LINKED_ROWS MAX_ROWS = LINKED_ROWS
PREPARED_SECONDS = 120 PREPARED_SECONDS = 120
_prepared: list = [] # [(created, session with the home page loaded)] _prepared: list = [] # [(created, session with the home page loaded)]
@ -139,12 +141,23 @@ def search(text: str, *, years: tuple[int, int] | None = None, work_types: list[
return {**result, "cached": False} return {**result, "cached": False}
def _expanded(session, first: dict, actions: dict) -> dict:
"""All values of the year and supervisor facets (two requests), exact over the whole result."""
full = {}
for title in EXPANDED:
if title in actions and len(first["facets"].get(title, [])) >= 10: # CRZP shows the top 10 only
full[title] = session.show_all(actions[title])
elif title in first["facets"]:
full[title] = first["facets"][title]
return full
def _search_live(text, years, work_types, links, session_factory, depth=LINKED_ROWS) -> dict: def _search_live(text, years, work_types, links, session_factory, depth=LINKED_ROWS) -> dict:
depth = max(depth, LINKED_ROWS)
with session_factory() as session: with session_factory() as session:
first = parse_results(session.search(text, years=years, work_types=work_types)) first = parse_results(session.search(text, years=years, work_types=work_types))
actions = facet_actions(getattr(session, "content", "") or "") # ids from the first result page
rows = first["rows"] rows = first["rows"]
if first["total"] > len(rows): # up to 20 results: one page, no paging requests if first["total"] > len(rows) and depth > len(rows): # up to 20 results: one page, no paging requests
rows = parse_rows(session.set_page_size()) rows = parse_rows(session.set_page_size())
while len(rows) < min(first["total"], depth): while len(rows) < min(first["total"], depth):
page = session.next_page() page = session.next_page()
@ -157,6 +170,8 @@ def _search_live(text, years, work_types, links, session_factory, depth=LINKED_R
rows = rows[:depth] rows = rows[:depth]
if rows and links: if rows and links:
_attach_links(rows[:LINKED_ROWS], parse_rss(session.rss_links())) _attach_links(rows[:LINKED_ROWS], parse_rss(session.rss_links()))
if first["total"] > len(first["rows"]): # last: after an expansion CRZP pages no more rows
first["facets"] = {**first["facets"], **_expanded(session, first, actions)}
return {"query": text, "years": list(years) if years else None, "work_types": work_types or None, return {"query": text, "years": list(years) if years else None, "work_types": work_types or None,
"total": first["total"], "facets": first["facets"], "rows": rows, "complete": len(rows) >= first["total"], "total": first["total"], "facets": first["facets"], "rows": rows, "complete": len(rows) >= first["total"],
"links": links} "links": links}
@ -177,3 +192,64 @@ def detail(sid: str, *, session_factory=CrzpSession, cache: CrzpStore | None = N
record["url"] = PERMALINK.format(sid=record["sid"]) record["url"] = PERMALINK.format(sid=record["sid"])
cache.put_detail(sid, record) cache.put_detail(sid, record)
return record return record
_chunk_pool = {"pool": None}
def _pool():
"""One shared pool for year-range reads (no new threads per search)."""
from concurrent.futures import ThreadPoolExecutor
with _store_lock:
if _chunk_pool["pool"] is None:
_chunk_pool["pool"] = ThreadPoolExecutor(max_workers=max(1, PARALLEL), thread_name_prefix="crzp-range")
return _chunk_pool["pool"]
def year_counts(result: dict) -> dict[int, int]:
return {int(f["label"]): f["count"] for f in result["facets"].get("Rok odovzdania", [])
if str(f["label"]).isdigit()}
def plan(counts: dict[int, int], years, cap: int, limit: int) -> list[tuple[int, int]]:
"""Year ranges, newest first, each with at most `cap` records (one year may exceed it),
until `limit` records are planned."""
low, high = years or (min(counts), max(counts))
chunks, current, size, planned = [], [], 0, 0
for year, count in sorted(((y, n) for y, n in counts.items() if low <= y <= high), reverse=True):
if planned >= limit:
break
if current and size + count > cap:
chunks.append((min(current), max(current)))
current, size = [], 0
current.append(year)
size += count
planned += count
if current:
chunks.append((min(current), max(current)))
return chunks
def search_deep(text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None,
links: bool = False, limit: int = DEEP_ROWS, cache: CrzpStore | None = None) -> dict:
"""Up to `limit` rows of a result larger than one read, read in year ranges
(the year facet tells how many records each year has), PARALLEL ranges at once.
With links, every range has at most LINKED_ROWS records, so its RSS export
gives a permalink to every row; without links the ranges are larger."""
head = search(text, years=years, work_types=work_types, links=links,
depth=LINKED_ROWS if links else 20, cache=cache)
if head["complete"]:
return head
counts = year_counts(head)
if not counts:
return search(text, years=years, work_types=work_types, links=links, depth=limit, cache=cache)
wanted = min(head["total"], limit)
cap = LINKED_ROWS if links else max(LINKED_ROWS, -(-wanted // PARALLEL))
chunks = plan(counts, years, cap, limit)
parts = list(_pool().map(lambda c: search(text, years=c, work_types=work_types, links=links,
depth=LINKED_ROWS if links else limit, cache=cache), chunks))
rows = [r for part in parts for r in part["rows"]][:limit]
read_total = sum(part["total"] for part in parts)
return {**head, "rows": rows, "complete": len(rows) >= head["total"], "chunks": chunks,
"links": links, "cached": all(part.get("cached") for part in parts),
"chunks_complete": all(part["complete"] for part in parts) and read_total >= head["total"]}

42
tools/crzp/source/client.py
View File

@ -6,8 +6,10 @@ replays exactly what the browser sends (no JavaScript engine is needed):
search with year and work-type filters, change the page size, go to the next page and export search with year and work-type filters, change the page size, go to the next page and export
the permalinks (RSS); a record detail is read through its public permalink. Only metadata pages are read; full texts and reviews never. the permalinks (RSS); a record detail is read through its public permalink. Only metadata pages are read; full texts and reviews never.
All requests of the process share one throttle (CRZP answers bursts with All requests of the process share one throttle: at most one request start per
HTTP 429), so concurrency cannot raise the request rate. CRZP_MIN_INTERVAL_SECONDS (0.5 s, i.e. 2 requests/s; one CRZP page takes about
1.1 s, so only reads of several year ranges at once overlap). An HTTP 429 slows
every request down to CRZP_PENALTY_SECONDS (2 s) for CRZP_PENALTY_DURATION (300 s).
""" """
from __future__ import annotations from __future__ import annotations
@ -33,14 +35,24 @@ class CrzpUnavailable(RuntimeError):
class _Throttle: class _Throttle:
"""Minimum interval between any two requests of this process.""" """Minimum interval between any two request starts of this process; slower after a 429."""
def __init__(self): def __init__(self):
self._lock = threading.Lock() self._lock = threading.Lock()
self._last = 0.0 self._last = 0.0
self._penalty_until = 0.0
def penalize(self):
self._penalty_until = time.monotonic() + float(os.getenv("CRZP_PENALTY_DURATION", "300"))
def interval(self) -> float:
interval = float(os.getenv("CRZP_MIN_INTERVAL_SECONDS", "0.5"))
if time.monotonic() < self._penalty_until:
interval = max(interval, float(os.getenv("CRZP_PENALTY_SECONDS", "2.0")))
return interval
def wait(self): def wait(self):
interval = float(os.getenv("CRZP_MIN_INTERVAL_SECONDS", "1.0")) interval = self.interval()
with self._lock: with self._lock:
delay = self._last + interval - time.monotonic() delay = self._last + interval - time.monotonic()
if delay > 0: if delay > 0:
@ -87,7 +99,7 @@ class CrzpSession:
"""One OPAC session (cookie + current page). Not shared between threads.""" """One OPAC session (cookie + current page). Not shared between threads."""
def __init__(self, timeout: float | None = None): def __init__(self, timeout: float | None = None):
timeout = timeout or float(os.getenv("CRZP_TIMEOUT_SECONDS", "20")) timeout = timeout or float(os.getenv("CRZP_TIMEOUT_SECONDS", "30"))
self.http = httpx.Client(base_url=BASE_URL, headers={"User-Agent": USER_AGENT}, self.http = httpx.Client(base_url=BASE_URL, headers={"User-Agent": USER_AGENT},
follow_redirects=True, timeout=timeout) follow_redirects=True, timeout=timeout)
self.page = "" # last full page: form target and hidden fields self.page = "" # last full page: form target and hidden fields
@ -115,11 +127,15 @@ class CrzpSession:
try: try:
response = self.http.request(method, url, **kwargs) response = self.http.request(method, url, **kwargs)
except httpx.HTTPError as error: except httpx.HTTPError as error:
raise CrzpUnavailable("network_error") from error if attempt == 2:
raise CrzpUnavailable("network_error") from error
time.sleep(1.0 + attempt) # one slow or dropped answer is repeated, not reported
continue
if response.status_code != 429: if response.status_code != 429:
if response.status_code >= 400: if response.status_code >= 400:
raise CrzpUnavailable(f"http_{response.status_code}") raise CrzpUnavailable(f"http_{response.status_code}")
return response return response
THROTTLE.penalize()
retry = response.headers.get("Retry-After", "") retry = response.headers.get("Retry-After", "")
time.sleep(min(float(retry) if retry.isdigit() else 3.0 * (attempt + 1), 10.0)) time.sleep(min(float(retry) if retry.isdigit() else 3.0 * (attempt + 1), 10.0))
raise CrzpUnavailable("rate_limited") raise CrzpUnavailable("rate_limited")
@ -199,6 +215,20 @@ class CrzpSession:
match = re.search(r"ae\(&#39;(\w+)&#39;,&#39;(navigator:navBiblio_\d+)&#39;\);\" title=\"\+1\"", self.content) match = re.search(r"ae\(&#39;(\w+)&#39;,&#39;(navigator:navBiblio_\d+)&#39;\);\" title=\"\+1\"", self.content)
return self.act(match.group(1), match.group(2)) if match else None return self.act(match.group(1), match.group(2)) if match else None
def show_all(self, element: str) -> list[dict]:
"""Every value of a facet ("Rok odovzdania", "Školiteľ"): [{label, count}] (one request)."""
script = self.act(element, "showMoreResultsNewAjax")
if "[" not in script: # right after another expansion CRZP first only renews the action GUID
script = self.act(element, "showMoreResultsNewAjax")
start = script.find("[")
try:
items = json.loads(script[start:script.rfind("]") + 1]) if start >= 0 else None
except ValueError:
items = None
if not isinstance(items, list) or not all(isinstance(i, dict) and "label" in i and "count" in i for i in items):
raise CrzpUnavailable("page_structure_changed")
return [{"label": str(i["label"]), "count": int(i["count"])} for i in items]
def rss_links(self) -> str: def rss_links(self) -> str:
"""RSS export of the current search: titles and permalinks (CRZP caps it at 200).""" """RSS export of the current search: titles and permalinks (CRZP caps it at 200)."""
element = action_element(self.content, "makeRSSChannelAjax") or action_element(self.page, "makeRSSChannelAjax") element = action_element(self.content, "makeRSSChannelAjax") or action_element(self.page, "makeRSSChannelAjax")

15
tools/crzp/source/parse.py
View File

@ -87,6 +87,21 @@ def facets(page: str) -> dict[str, list[dict]]:
return result return result
def facet_actions(page: str) -> dict[str, str]:
"""Facet title -> id of the element that shows all of its values."""
result = {}
for raw in re.findall(r'<script type="application/json" id="props-\w+">(.*?)</script>', page, re.S):
try:
data = json.loads(raw.replace('\\\\"', '\\"'))
except ValueError:
continue
show = data.get("onShowAll") if isinstance(data, dict) else None
if (isinstance(data.get("strings") if isinstance(data, dict) else None, dict) and isinstance(show, list)
and len(show) == 2 and isinstance(show[1], list) and show[1][1:2] == ["showMoreResultsNewAjax"]):
result[data["strings"].get("title", "")] = show[1][0]
return result
def _lines(chunk: str, label: str) -> list[str]: def _lines(chunk: str, label: str) -> list[str]:
return [text(v) for v in re.findall(re.escape(label) + r":\s*([^<]*)<", chunk) if text(v)] return [text(v) for v in re.findall(re.escape(label) + r":\s*([^<]*)<", chunk) if text(v)]

14
tools/crzp/storage/cache.py
View File

@ -75,6 +75,20 @@ class CrzpStore:
def put_detail(self, sid: str, payload: dict) -> None: def put_detail(self, sid: str, payload: dict) -> None:
self._put("details", sid, payload) self._put("details", sid, payload)
def cached_rows(self, ttl: float | None = None):
"""Every row of every cached search (records read before; used for the workplace and
vocabulary tables). Rows of an older schema are skipped."""
if not self.path.exists():
return
with self._connect() as connection:
items = connection.execute("SELECT fetched_at, payload FROM searches").fetchall()
for fetched_at, payload in items:
if ttl is not None and time.time() - fetched_at > ttl:
continue
value = json.loads(payload)
if value.get("schema") == SCHEMA:
yield from value.get("rows", [])
def stats(self) -> dict: def stats(self) -> dict:
if not self.path.exists(): if not self.path.exists():
return {"searches": 0, "details": 0} return {"searches": 0, "details": 0}

75
tools/crzp/storage/graph.py
View File

@ -45,6 +45,77 @@ RETURN o.sid AS sid, collect(k.label) AS shared ORDER BY size(shared) DESC, sid
""" """
# Links to the ZP Wiki graph, written by code only (never by the model):
# a CRZP person with a work at TUKE whose name equals a ZP Wiki person (probably the same person; CRZP
# has no identifiers), and a CRZP keyword that is a ZP Wiki topic or keyword (same words or reviewed synonyms).
LINK_PEOPLE = """
UNWIND $pairs AS pair
MATCH (c:CrzpPerson {name: pair.crzp})
MATCH (p:Person {name: pair.wiki})
MERGE (c)-[r:PROBABLY_SAME]->(p) SET r.basis = 'rovnaké meno, práca na TUKE'
"""
LINK_TOPICS = """
UNWIND $pairs AS pair
MATCH (k:CrzpKeyword {key: pair.key})
MATCH (t) WHERE (t:Topic OR t:Keyword) AND t.name = pair.wiki
MERGE (k)-[:SAME_TOPIC]->(t)
"""
WIKI_NAMES = "MATCH (p:Person) RETURN p.name AS name"
WIKI_TOPICS = "MATCH (t) WHERE t:Topic OR t:Keyword RETURN DISTINCT t.name AS name"
PERSON_WORKS = """
MATCH (p:Person {name: $name})<-[:PROBABLY_SAME]-(c:CrzpPerson)<-[r:AUTHOR|SUPERVISOR|OPPONENT]-(t:CrzpThesis)
RETURN t.sid AS sid, t.title AS title, t.year AS year, t.work_type AS work_type, type(r) AS role ORDER BY t.year DESC
"""
_wiki = {"at": 0.0, "names": [], "topics": []}
def link_pairs(rows: list[dict], wiki_names: list[str], wiki_topics: list[str]) -> tuple[list[dict], list[dict]]:
"""(person pairs, keyword pairs) for the rows; pure, so it is tested without Neo4j."""
from tools.crzp.source.parse import display_name
from tools.retrieval.synonyms import base_name
names = {fold(n): n for n in wiki_names if n}
topics = {}
for topic in wiki_topics:
if topic:
topics.setdefault(fold(topic), topic)
topics.setdefault(fold(base_name(topic)), topic)
people, keywords = {}, {}
for row in rows:
if "technicka univerzita v kosiciach" not in fold(row.get("school") or ""):
continue
for field in ("authors", "supervisors", "opponents"):
for name in row.get(field, []):
wiki = names.get(fold(display_name(name)))
if wiki:
people[name] = wiki
for row in rows:
for keyword in row.get("keywords", []):
wiki = topics.get(fold(keyword)) or topics.get(fold(base_name(keyword)))
if wiki:
keywords[fold(keyword)] = wiki
return ([{"crzp": c, "wiki": w} for c, w in people.items()], [{"key": k, "wiki": w} for k, w in keywords.items()])
def link(session, rows: list[dict]) -> None:
import time
if time.time() - _wiki["at"] > 600:
_wiki.update(at=time.time(), names=session.run(WIKI_NAMES).value(), topics=session.run(WIKI_TOPICS).value())
people, keywords = link_pairs(rows, _wiki["names"], _wiki["topics"])
if people:
session.execute_write(lambda tx: tx.run(LINK_PEOPLE, pairs=people).consume())
if keywords:
session.execute_write(lambda tx: tx.run(LINK_TOPICS, pairs=keywords).consume())
def person_works(name: str) -> list[dict]:
"""CRZP works linked to a ZP Wiki person in the graph ([] without Neo4j)."""
try:
with _session() as session:
return [dict(r) for r in session.run(PERSON_WORKS, name=name)]
except Exception:
return []
def _session(): def _session():
global _driver global _driver
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
@ -66,6 +137,10 @@ def project(rows: list[dict]) -> bool:
try: try:
with _session() as session: with _session() as session:
session.execute_write(lambda tx: tx.run(PROJECT, rows=data).consume()) session.execute_write(lambda tx: tx.run(PROJECT, rows=data).consume())
try:
link(session, rows)
except Exception:
logger.warning("Linking CRZP records to the ZP Wiki graph failed.")
return True return True
except Exception: except Exception:
logger.warning("CRZP projection to Neo4j failed; Python aggregation is used.") logger.warning("CRZP projection to Neo4j failed; Python aggregation is used.")

98
tools/crzp/tests/test_crzp.py
View File

@ -195,12 +195,16 @@ def test_about_shows_a_summary_only_after_the_check(crzp):
"Autor meral teplotu senzormi ESP32 a údaje posielal cez MQTT.", '{"supported": true}']) "Autor meral teplotu senzormi ESP32 a údaje posielal cez MQTT.", '{"supported": true}'])
model = lambda messages, **kw: {"choices": [{"message": {"content": next(replies)}}]} model = lambda messages, **kw: {"choices": [{"message": {"content": next(replies)}}]}
result = A._answer("O čom je práca Jána Kováča?", complete=model) result = A._answer("O čom je práca Jána Kováča?", complete=model)
assert "Zhrnutie podľa abstraktu: Autor meral teplotu" in result["answer"] and "54 strán" in result["answer"] assert "\n\nAutor meral teplotu senzormi ESP32" in result["answer"] and "54 strán" not in result["answer"]
assert result["answer"].startswith("**Meranie teploty cez MQTT** — bakalárska práca, 2025, Technická univerzita v Košiciach; autor Ján Kováč")
A._cache.clear() A._cache.clear()
replies = iter(['{"people":[{"name":"Ján Kováč","role":"author"}],"topics":[],"title":null,"target_role":null}', replies = iter(['{"people":[{"name":"Ján Kováč","role":"author"}],"topics":[],"title":null,"target_role":null}',
"Autor vyvinul blockchain.", '{"supported": false}']) "Autor vyvinul blockchain.", '{"supported": false}', "Autor vyvinul kvantový počítač.",
'{"supported": false}'])
rejected = A._answer("O čom je práca Jána Kováča?", complete=model) rejected = A._answer("O čom je práca Jána Kováča?", complete=model)
assert "Abstrakt: Práca opisuje meranie teploty" in rejected["answer"] and "blockchain" not in rejected["answer"] # Never the copied abstract: a sentence from the CRZP keywords instead of an unsupported summary.
assert "Bakalárska práca „Meranie teploty cez MQTT“ sa zaoberá témami: iot, mqtt." in rejected["answer"]
assert "blockchain" not in rejected["answer"] and "Práca opisuje meranie teploty" not in rejected["answer"]
def test_relation_aggregates_opponents_of_supervised_works(crzp): def test_relation_aggregates_opponents_of_supervised_works(crzp):
@ -251,13 +255,13 @@ def test_client_retries_rate_limit_then_reports_it(monkeypatch):
lambda request: httpx.Response(429, text="slow down"))) lambda request: httpx.Response(429, text="slow down")))
with pytest.raises(C.CrzpUnavailable, match="rate_limited"): with pytest.raises(C.CrzpUnavailable, match="rate_limited"):
session.get("/") session.get("/")
assert C.THROTTLE.interval() >= 2.0 # a 429 slows every next request down
C.THROTTLE._penalty_until = 0.0
def test_crzp_is_an_answer_engine_of_the_mcp_client(): def test_crzp_is_no_separate_tool_of_the_mcp_client():
import packages.claude.client as client import packages.claude.client as client
assert "crzp" in client.ENGINES assert client.ENGINES == ("rag", "graphrag") # CRZP answers inside /rag and /graphrag
assert client.query_backend("/rag Práca?", "crzp")["reason"] == "engine_mismatch"
assert client.query_backend("Práca?", "crzp", mode="retrieve")["reason"] == "invalid_query"
# ---------------------------------------------------------------- real page structure (anonymized fixtures) # ---------------------------------------------------------------- real page structure (anonymized fixtures)
@ -353,7 +357,7 @@ def test_verified_workplace_filter_and_label(crzp):
assert "KEMT práca" in result["answer"] and "Iná katedra" not in result["answer"] assert "KEMT práca" in result["answer"] and "Iná katedra" not in result["answer"]
assert "Technická univerzita v Košiciach, FEI, KEMT" in result["answer"] assert "Technická univerzita v Košiciach, FEI, KEMT" in result["answer"]
assert units.unit_label({"school": "Technická univerzita v Košiciach", "school_unit": "1040 104009"}) == \ assert units.unit_label({"school": "Technická univerzita v Košiciach", "school_unit": "1040 104009"}) == \
"FEI, pracovisko 104009" "FEI, Katedra počítačov a informatiky (104009)"
assert units.unit_in_question("práce z FEI") == "fei" and units.unit_in_question("práce z TUKE") is None assert units.unit_in_question("práce z FEI") == "fei" and units.unit_in_question("práce z TUKE") is None
@ -389,9 +393,9 @@ def test_structure_check_reports_degraded_health(monkeypatch, tmp_path):
def test_department_written_by_name_is_kemt_and_not_a_namesake(crzp): def test_department_written_by_name_is_kemt_and_not_a_namesake(crzp):
assert split_school("Technická univerzita v Košiciach 1040 Katedra elektroniky a multimediálnych telekomunikácií") == \ assert split_school("Technická univerzita v Košiciach 1040 Katedra elektroniky a multimediálnych telekomunikácií") == \
("Technická univerzita v Košiciach", "1040 Katedra elektroniky a multimediálnych telekomunikácií") ("Technická univerzita v Košiciach", "1040 Katedra elektroniky a multimediálnych telekomunikácií")
old = row("Stará", author="Galajda, Pavol", old = row("Stará", author="Galajda, Pavol", year=2012,
school="Technická univerzita v Košiciach 1040 Katedra elektroniky a multimediálnych telekomunikácií") school="Technická univerzita v Košiciach 1040 Katedra elektroniky a multimediálnych telekomunikácií")
new = row("Nová", author="Galajda, Pavol") new = row("Nová", author="Galajda, Pavol", kind="Zverejnená diplomová práca", year=2014)
crzp["result"] = found([old, new]) crzp["result"] = found([old, new])
result = A._answer("Ktoré práce na KEMT napísal Pavol Galajda?", complete=crzp["no_model"]) result = A._answer("Ktoré práce na KEMT napísal Pavol Galajda?", complete=crzp["no_model"])
assert "**Stará**" in result["answer"] and "**Nová**" in result["answer"] assert "**Stará**" in result["answer"] and "**Nová**" in result["answer"]
@ -486,7 +490,7 @@ def test_school_count_uses_the_crzp_total_only_when_every_read_row_is_from_that_
dp = dict(kind="Zverejnená diplomová práca", year=2024) dp = dict(kind="Zverejnená diplomová práca", year=2024)
crzp["result"] = found([row("A", **dp), row("B", **dp)], total=1169) crzp["result"] = found([row("A", **dp), row("B", **dp)], total=1169)
result = A._answer("Koľko diplomových prác bolo na TUKE v roku 2024?", complete=crzp["no_model"]) result = A._answer("Koľko diplomových prác bolo na TUKE v roku 2024?", complete=crzp["no_model"])
assert "**1169** (súhrn CRZP pre hľadanie názvu školy" in result["answer"] assert "**1169** (súhrnný počet z CRZP)" in result["answer"] and not result["truncated"]
crzp["result"] = found([row("A", **dp), row("B", school="Iná škola", **dp)], total=1169) crzp["result"] = found([row("A", **dp), row("B", school="Iná škola", **dp)], total=1169)
A._cache.clear() A._cache.clear()
mixed = A._answer("Koľko diplomových prác bolo na TUKE v roku 2024?", complete=crzp["no_model"]) mixed = A._answer("Koľko diplomových prác bolo na TUKE v roku 2024?", complete=crzp["no_model"])
@ -645,7 +649,7 @@ def test_old_department_name_deep_reads_and_rule_phrases(crzp):
r["sid"] = key[:6] r["sid"] = key[:6]
result = A._answer("Koľko prác bolo na KEMT v roku 2012?", complete=crzp["no_model"]) result = A._answer("Koľko prác bolo na KEMT v roku 2012?", complete=crzp["no_model"])
assert "**2**" in result["answer"] and "Hľadané výrazy" not in result["answer"] assert "**2**" in result["answer"] and "Hľadané výrazy" not in result["answer"]
assert {f["depth"] for _, f in crzp["searches"]} == {fetch.DEEP_ROWS} # counts read deeper than 200 rows assert {f["depth"] for _, f in crzp["searches"]} == {20} # counted by CRZP: first page and facets only
query = understand("Ktoré práce sa zaoberali rozpoznávaním reči?", use_model=False) query = understand("Ktoré práce sa zaoberali rozpoznávaním reči?", use_model=False)
assert query.phrases[0].endswith(" reči") and A.search_groups(query)[0][0] == query.phrases[0] assert query.phrases[0].endswith(" reči") and A.search_groups(query)[0][0] == query.phrases[0]
from tools.crzp.answering.understand import topic_phrases from tools.crzp.answering.understand import topic_phrases
@ -690,4 +694,72 @@ def test_which_works_dealt_with_x_is_a_list_and_one_work_is_summarized_once(crzp
crzp["result"] = found([row("Moja práca", author="Pták, Ján"), row("Moja práca", author="Pták, Ján")]) crzp["result"] = found([row("Moja práca", author="Pták, Ján"), row("Moja práca", author="Pták, Ján")])
crzp["detail"] = parse_detail(DETAIL) crzp["detail"] = parse_detail(DETAIL)
result = A._answer("O čom je práca Jána Ptáka?", complete=crzp["no_model"]) result = A._answer("O čom je práca Jána Ptáka?", complete=crzp["no_model"])
assert result["answer"].count("Zhrnutie podľa abstraktu") + result["answer"].count("Abstrakt:") == 1 assert result["answer"].count("**Moja práca**") == 1
def test_two_bachelor_theses_of_one_name_are_two_students_and_a_continuation_is_hinted():
rows = found([row("A", author="Novák, Ján", year=2012), row("B", author="Novák, Ján", year=2020),
row("C", author="Novák, Ján", kind="Zverejnená diplomová práca", year=2014),
row("D", author="Novák, Ján", kind="Zverejnená diplomová práca", year=2022,
school="Žilinská univerzita v Žiline")])["rows"]
groups = A.identities(rows, Person(name="Ján Novák", role="author"))
tuke = [g for g in groups if g["school"] == "Technická univerzita v Košiciach"]
assert sorted(sorted(r["title"] for r in g["rows"]) for g in tuke) == [["A", "C"], ["B"]]
zilina = next(g for g in groups if g["school"] == "Žilinská univerzita v Žiline")
assert zilina["hint"] == "možno tá istá osoba ako Technická univerzita v Košiciach (2020)"
def test_learned_workplace_names_and_acronyms(monkeypatch):
from tools.crzp.answering import workplaces
assert workplaces.acronym("Katedra počítačov a informatiky") == "KPI"
assert workplaces._department("1040 Katedra počítačov a informatiky") == "Katedra počítačov a informatiky"
assert workplaces._department("1040 104009") is None
def search(text, **kw):
if text == "104009":
return {"facets": {"Školiteľ": [{"label": "Korečko, Štefan", "count": 9}, {"label": "Nový, Adam", "count": 5},
{"label": "Malý, Jozef", "count": 4}, {"label": "Iný, Peter", "count": 3}]},
"rows": []}
unit = {"Štefan Korečko": "1040 Katedra počítačov a informatiky", "Adam Nový": "1040 Katedra počítačov a informatiky",
"Jozef Malý": "1040 Katedra počítačov a inform", "Peter Iný": "1040 Katedra matematiky"}[text]
return {"facets": {}, "rows": [{"school": "Technická univerzita v Košiciach", "school_unit": unit,
"supervisors": [text.split()[1] + ", " + text.split()[0]], "title": "x"}]}
outcome = workplaces.learn_code("104009", search=search)
assert outcome["accepted"] and outcome["name"] == "Katedra počítačov a informatiky" and outcome["voters"] == 4
assert outcome["votes"]["Katedra počítačov a informatiky"] == 3 # the cut spelling counts as the full name
def test_an_incomplete_wiki_profile_is_supplemented_from_crzp_for_that_person_only(crzp, monkeypatch):
from tools.crzp.answering import wiki_bridge
crzp["result"] = found([row("Moja práca", author="Pták, Ján", opponents=("Hládek, Daniel",)),
row("Cudzia", author="Pták, Ján", school="Iná univerzita v Trnave")])
monkeypatch.setattr(wiki_bridge, "wiki_person", lambda q, db_file=None: "Ján Pták")
wiki = {"status": "no_answer", "reason": "missing_evidence", "accepted_for_delivery": True,
"answer": "V ZP Wiki som oponenta nenašiel.", "source_urls": [], "claims": [], "index_version": {"version": "v"}}
question = "Kto bol oponent bakalárskej práce Jána Ptáka?"
result = wiki_bridge.supplement(question, wiki, "rag", complete=crzp["no_model"])
assert result["status"] == "ok" and result["answer"].startswith("V ZP Wiki som oponenta nenašiel.")
assert "**Doplnené z CRZP**" in result["answer"] and "Daniel Hládek" in result["answer"]
assert "Cudzia" not in result["answer"] # only Technická univerzita v Košiciach
import packages.claude.client as client
client.validate_metadata(result)
assert client.validated_delivery_check(result, question, "rag")
assert wiki_bridge.supplement("Aké výsledky dosiahol Ján Pták?", wiki, "rag", complete=crzp["no_model"]) is None
assert wiki_bridge.supplement("Obhájil Ján Pták bakalársku prácu?", wiki, "rag", complete=crzp["no_model"]) is None
monkeypatch.setattr(wiki_bridge, "wiki_person", lambda q, db_file=None: None)
assert wiki_bridge.supplement(question, wiki, "rag", complete=crzp["no_model"]) is None # not a wiki person
def test_rule_mode_uses_the_tag_crzp_records_carry(tmp_path, monkeypatch):
from tools.crzp.answering import lexicon
store = CrzpStore(tmp_path / "c.sqlite")
store.put_search("x", {"query": "x", "total": 2, "facets": {}, "complete": True, "links": True, "rows": [
{"title": "A", "keywords": ["rozpoznávanie emócií", "reč"]}, {"title": "B", "keywords": ["Rozpoznávanie emócií"]}]})
monkeypatch.setattr(fetch, "store", lambda: store)
monkeypatch.setitem(lexicon._state, "built", 0.0)
assert lexicon.tag_for(["rozpoznávaním", "emócií"]) == "rozpoznávanie emócií"
assert lexicon.tag_for(["rozpoznávaním", "tváre"]) is None
monkeypatch.setitem(lexicon._state, "built", 0.0)
def test_a_code_without_learned_evidence_stays_a_number():
assert units.unit_label({"school": "Technická univerzita v Košiciach", "school_unit": "1040 104003"}) == "FEI, pracovisko 104003"

23
tools/knowledge/academic.py
View File

@ -491,16 +491,19 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None,
notes=[r for r in notes if set(plan.spec.facets)&set(facets_for(r['text'], notes=[r for r in notes if set(plan.spec.facets)&set(facets_for(r['text'],
{'assignment':'assignment','planned':'task','result':'progress','progress':'progress'}.get(r['phase'],'other'), {'assignment':'assignment','planned':'task','result':'progress','progress':'progress'}.get(r['phase'],'other'),
plan.spec.question,r['phase']))] plan.spec.question,r['phase']))]
notes.sort( if plan and plan.spec.facets:
key=lambda r: ( notes.sort(
-unit_score( key=lambda r: (
r["text"], -unit_score(
"progress" if r["phase"] in {"result", "progress"} else "task", r["text"],
), "progress" if r["phase"] in {"result", "progress"} else "task",
r["phase"] != "result", ),
r["line"], r["phase"] != "result",
r["line"],
)
) )
) else: # the whole meeting: its status and then its tasks, in the order of the record
notes.sort(key=lambda r: (r["phase"] == "planned", r["line"]))
actual = [r for r in notes if r["phase"] in {"result", "progress"}] actual = [r for r in notes if r["phase"] in {"result", "progress"}]
planned = [r for r in notes if r["phase"] == "planned"] planned = [r for r in notes if r["phase"] == "planned"]
selected = actual[:4] + planned[:4] selected = actual[:4] + planned[:4]
@ -528,7 +531,7 @@ def answer_academic(question, index, *, engine, limit=5, units=None, mode=None,
phase = { phase = {
"result": "Zaznamenaný výsledok", "result": "Zaznamenaný výsledok",
"progress": "Priebežný stav", "progress": "Priebežný stav",
"planned": "Plán", "planned": "Úloha zo stretnutia",
"assignment": "Zadanie", "assignment": "Zadanie",
}.get(note["phase"], "Záznam") }.get(note["phase"], "Záznam")
quote = re.sub(r"^\s*(?:[-*+]|\d+[.)])\s+", "", note["text"]) quote = re.sub(r"^\s*(?:[-*+]|\d+[.)])\s+", "", note["text"])

57
tools/knowledge/planned_answers.py
View File

@ -9,6 +9,56 @@ from tools.answers.content_selection import facets_for, unit_score
from tools.knowledge.evidence import claim_eligible from tools.knowledge.evidence import claim_eligible
WORK_IN = {'diploma_thesis':'vo svojej diplomovej práci','diploma_project':'vo svojom diplomovom projekte',
'bachelor_thesis':'vo svojej bakalárskej práci','bachelor_project':'vo svojom bakalárskom projekte'}
NOT_TOOLS = {'zpwiki','zp wiki','kemt','tuke','fei','hpc','lr','sk','hf','eval','loss','dp','bp','ok'}
TOOL_NOUNS = {'server','model','modely','modelu','dataset','datasety','databaza','databazy','framework','kniznica',
'collection','api','agent','bot','chatbot','kontajner','sluzba','rozhranie','tools','tool'}
def _tool_concept(concept):
""""MCP server", "Fast120 (Qwen3.5)", "AYA collection dataset" — not "prístup na HPC" or a URL."""
from tools.answers.content_selection import named_tools
if re.search(r'https?:|/|\.[a-z]{2,4}\b',concept) or len(concept.split())>4:return False
tools={normalized(t) for t in named_tools(concept)}-NOT_TOOLS
words=[w for w in normalized(concept).split()]
return bool(tools) and all(w in tools or w in TOOL_NOUNS or any(w in normalized(t).split() for t in tools)
for w in words)
METHOD_PHRASE = re.compile(r'(?i)\b(?:vektorov\w*\s+a\s+textov\w*|vektorov\w*|textov\w*|sémantick\w*|full-?text\w*)\s+vyhľadáv\w*'
r'|\b(?:dotrénovani\w*|fine-?tuning\w*|znalostn\w* graf\w*|grafov\w* databáz\w*)')
def methods_sentence(person, label, rows, candidates):
""""Ján Pták vo svojej diplomovej práci „Agent…“ využíva: RAG, MCP server, …" from the
tools and methods the cited records name (concepts of those records, named tools, method phrases)."""
from tools.answers.content_selection import named_tools
found=[];starting=set()
for row in candidates:
text=re.sub(r'https?://\S+','',re.sub(r'^\s*(?:[-*+]\s+|\d+[.)]\s+)','',row['text']))
begins=bool(re.search(r'\b(?:zacin\w*|zacal\w*|skusa\w*|vyskusa\w*|testuj\w*)\b',normalized(text)))
before=len(found)
for phrase in METHOD_PHRASE.findall(text):found.append(phrase.strip())
for concept in row.get('concepts') or []:
if isinstance(concept,str) and (_tool_concept(concept) or METHOD_PHRASE.fullmatch(concept.strip())):
found.append(concept.strip())
found+=[t for t in named_tools(text) if not re.search(r'\.[a-z]{2,4}$|/',t) and normalized(t) not in NOT_TOOLS
and (len(t)>=3 or t.lower() in NOT_TOOLS)]
if begins:starting|={normalized(t) for t in found[before:]} # "začíname pracovať na GraphRAG" is no use yet
tools=[]
for name in found:
key=normalized(name)
if not key or key in NOT_TOOLS or any(f' {key} ' in f' {normalized(t)} ' for t in tools):continue
tools=[t for t in tools if f' {normalized(t)} ' not in f' {key} '] # "MCP" inside "MCP server"
tools.append(name)
if not tools:return None
topic=next((re.sub(r'^\s*(?:Téma|Názov)\s*:\s*','',r['text']).strip() for r in rows if r['kind']=='topic'
and not r['text'].strip().endswith(':') and not r['text'].lstrip().startswith('#')),None)
kind=next((r.get('work_type') for r in rows if r.get('work_type')),None)
where=WORK_IN.get(kind,'vo svojej práci')+' ('+label.split()[-1]+')' if not topic else WORK_IN.get(kind,'vo svojej práci')+' „'+topic+'“'
tools=[t+(' (začína)' if normalized(t) in starting else '') for t in tools]
return person+' '+where+' podľa záznamov využíva: '+', '.join(tools)+'.'
def answer_facets(bound, index, units, *, engine, limit): def answer_facets(bound, index, units, *, engine, limit):
plan=bound.spec plan=bound.spec
people=[p for p in plan.people if p.role=='student'] people=[p for p in plan.people if p.role=='student']
@ -103,6 +153,13 @@ def answer_facets(bound, index, units, *, engine, limit):
distinct={} distinct={}
for row in candidates:distinct.setdefault(normalized(row['text']),row) for row in candidates:distinct.setdefault(normalized(row['text']),row)
candidates=list(distinct.values()) candidates=list(distinct.values())
if facet=='methods' and personal_facets[person]==['methods'] and candidates and plan.presentation not in {'comparison','table'}:
sentence=methods_sentence(person,label,groups[key],candidates)
if sentence:
lines.append(sentence);cells[(key,facet)].append(sentence)
for row in candidates:
if row['id'] not in used:evidence.append(row);used.add(row['id'])
continue
lines.append(FACET_LABELS[facet]+':') lines.append(FACET_LABELS[facet]+':')
if not candidates: if not candidates:
notice=person+' — '+label+': '+FACET_LABELS[facet].lower()+': v podkladoch chýbajú doložené údaje.' notice=person+' — '+label+': '+FACET_LABELS[facet].lower()+': v podkladoch chýbajú doložené údaje.'

9
tools/knowledge/roster.py
View File

@ -281,9 +281,12 @@ def profile_tags(db_file=None) -> dict:
def answer_related(spec, index, *, engine: str, db_file=None) -> dict: def answer_related(spec, index, *, engine: str, db_file=None) -> dict:
from collections import Counter from collections import Counter
asked = related_topic(spec) asked = related_topic(spec)
from tools.retrieval.synonyms import variants_pattern, base_name
# "súvisia s rozpoznávaním reči": every case form of the topic and of its reviewed
# synonyms ("speech recognition", "ASR"), not only the literal.
stem = topic_stem(asked) stem = topic_stem(asked)
# "súvisia s chatbotom": every case form of the topic, not only the literal. single = re.compile(r"(?<![\w-])" + re.escape(stem) + r"\w{0,4}(?![\w-])")
word = re.compile(r"(?<![\w-])" + re.escape(stem) + r"\w{0,4}(?![\w-])") word = variants_pattern(asked) if " " in asked or base_name(asked) != asked else single
tags = profile_tags(db_file) tags = profile_tags(db_file)
tag_names = {t for values in tags.values() for t in values if word.fullmatch(t)} tag_names = {t for values in tags.values() for t in values if word.fullmatch(t)}
key = min(tag_names, key=len) if tag_names else asked key = min(tag_names, key=len) if tag_names else asked
@ -291,6 +294,8 @@ def answer_related(spec, index, *, engine: str, db_file=None) -> dict:
# of the matching tag ("chatbotom" -> "chatbot"). # of the matching tag ("chatbotom" -> "chatbot").
spelled = original_phrase(spec.question, asked) spelled = original_phrase(spec.question, asked)
topic = spelled if normalized(spelled) == key else key topic = spelled if normalized(spelled) == key else key
if base_name(asked) != asked:
topic = base_name(asked) # "rozpoznávaním reči" -> "rozpoznávanie reči"
units = [r for r in index["units"] if claim_eligible(r) and r.get("work_label") and not administrative_only(r["text"])] units = [r for r in index["units"] if claim_eligible(r) and r.get("work_label") and not administrative_only(r["text"])]
mentions = [r for r in units if word.search(normalized(r["text"])) mentions = [r for r in units if word.search(normalized(r["text"]))
or any(word.fullmatch(normalized(c)) for c in r.get("concepts", []))] or any(word.fullmatch(normalized(c)) for c in r.get("concepts", []))]

83
tools/retrieval/synonyms.py Normal file
View File

@ -0,0 +1,83 @@
"""Reviewed topic synonyms shared by ZP Wiki (RAG, GraphRAG) and CRZP.
One group is one meaning; the first entry is the base Slovak name shown in
answers. Matching ignores case, diacritics and Slovak case endings
(phrase_pattern), so "rozpoznávaním reči", "speech recognition" and "ASR"
all find the same topic. Groups are reviewed by hand; no synonym is guessed.
"""
from __future__ import annotations
import re
from tools.answers.context_policy import normalized
GROUPS = [
["rozpoznávanie reči", "speech recognition", "ASR", "automatické rozpoznávanie reči", "prepis reči", "speech to text"],
["syntéza reči", "speech synthesis", "text to speech", "TTS"],
["chatbot", "dialógový systém", "konverzačný agent", "conversational agent"],
["veľký jazykový model", "LLM", "large language model", "jazykový model", "language model"],
["RAG", "retrieval augmented generation", "vyhľadávanie s generovaním"],
["znalostný graf", "knowledge graph"],
["GraphRAG", "graph RAG"],
["strojový preklad", "machine translation", "neurónový strojový preklad"],
["sumarizácia", "summarization", "sumarizácia textu", "zhrnutie textu"],
["rozpoznávanie pomenovaných entít", "NER", "named entity recognition"],
["analýza sentimentu", "sentiment analysis"],
["nenávistná reč", "hate speech", "toxický jazyk", "toxicita", "toxic language"],
["neurónová sieť", "neural network", "neurónové siete"],
["hlboké učenie", "deep learning"],
["strojové učenie", "machine learning"],
["umelá inteligencia", "artificial intelligence", "AI"],
["spracovanie prirodzeného jazyka", "NLP", "natural language processing"],
["počítačové videnie", "computer vision"],
["internet vecí", "IoT", "internet of things"],
["kybernetická bezpečnosť", "cybersecurity", "cyber security"],
["vektorové vyhľadávanie", "vector search", "sémantické vyhľadávanie", "semantic search"],
["oprava gramatiky", "grammatical error correction", "korekcia textu", "oprava interpunkcie"],
["generovanie otázok", "question generation"],
["rozpoznávanie emócií", "emotion recognition"],
]
_ENDING = r"\w{0,4}"
def _key(text: str) -> str:
return " ".join(normalized(text).split())
def _stem(word: str) -> str:
""""rozpoznavanim" -> "rozpoznavan"; short words and acronyms stay whole."""
if len(word) < 5 or not word.isalpha():
return word
for ending in ("ami", "ach", "och", "ovi", "ych", "ymi", "eho", "emu", "iou", "ou", "om", "ov", "ej", "ia", "ie",
"iu", "im", "y", "a", "e", "i", "u", "o"):
if word.endswith(ending) and len(word) - len(ending) >= 4:
return word[: -len(ending)]
return word
def phrase_pattern(text: str) -> str:
"""A regex (over normalized text) for the phrase in any Slovak case form."""
words = _key(text).split()
if not words:
return r"(?!)"
parts = [re.escape(_stem(w)) + (_ENDING if len(w) >= 5 and w.isalpha() else "") for w in words]
return r"(?<![\w-])" + r"\s+".join(parts) + r"(?![\w-])"
def group_of(text: str) -> list[str]:
"""The reviewed group the phrase belongs to (any case form), else [text]."""
key = _key(text)
for group in GROUPS:
if any(re.fullmatch(phrase_pattern(item), key) for item in group):
return group
return [text]
def base_name(text: str) -> str:
group = group_of(text)
return group[0] if len(group) > 1 else text
def variants_pattern(text: str) -> re.Pattern:
"""One regex matching the topic and its reviewed synonyms in any case form."""
return re.compile("|".join(phrase_pattern(item) for item in group_of(text)))