84 lines
3.6 KiB
Python
84 lines
3.6 KiB
Python
"""Reviewed topic synonyms shared by ZP Wiki (RAG, GraphRAG) and CRZP.
|
|
|
|
One group is one meaning; the first entry is the base Slovak name shown in
|
|
answers. Matching ignores case, diacritics and Slovak case endings
|
|
(phrase_pattern), so "rozpoznávaním reči", "speech recognition" and "ASR"
|
|
all find the same topic. Groups are reviewed by hand; no synonym is guessed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from tools.answers.context_policy import normalized
|
|
|
|
GROUPS = [
|
|
["rozpoznávanie reči", "speech recognition", "ASR", "automatické rozpoznávanie reči", "prepis reči", "speech to text"],
|
|
["syntéza reči", "speech synthesis", "text to speech", "TTS"],
|
|
["chatbot", "dialógový systém", "konverzačný agent", "conversational agent"],
|
|
["veľký jazykový model", "LLM", "large language model", "jazykový model", "language model"],
|
|
["RAG", "retrieval augmented generation", "vyhľadávanie s generovaním"],
|
|
["znalostný graf", "knowledge graph"],
|
|
["GraphRAG", "graph RAG"],
|
|
["strojový preklad", "machine translation", "neurónový strojový preklad"],
|
|
["sumarizácia", "summarization", "sumarizácia textu", "zhrnutie textu"],
|
|
["rozpoznávanie pomenovaných entít", "NER", "named entity recognition"],
|
|
["analýza sentimentu", "sentiment analysis"],
|
|
["nenávistná reč", "hate speech", "toxický jazyk", "toxicita", "toxic language"],
|
|
["neurónová sieť", "neural network", "neurónové siete"],
|
|
["hlboké učenie", "deep learning"],
|
|
["strojové učenie", "machine learning"],
|
|
["umelá inteligencia", "artificial intelligence", "AI"],
|
|
["spracovanie prirodzeného jazyka", "NLP", "natural language processing"],
|
|
["počítačové videnie", "computer vision"],
|
|
["internet vecí", "IoT", "internet of things"],
|
|
["kybernetická bezpečnosť", "cybersecurity", "cyber security"],
|
|
["vektorové vyhľadávanie", "vector search", "sémantické vyhľadávanie", "semantic search"],
|
|
["oprava gramatiky", "grammatical error correction", "korekcia textu", "oprava interpunkcie"],
|
|
["generovanie otázok", "question generation"],
|
|
["rozpoznávanie emócií", "emotion recognition"],
|
|
]
|
|
_ENDING = r"\w{0,4}"
|
|
|
|
|
|
def _key(text: str) -> str:
|
|
return " ".join(normalized(text).split())
|
|
|
|
|
|
def _stem(word: str) -> str:
|
|
""""rozpoznavanim" -> "rozpoznavan"; short words and acronyms stay whole."""
|
|
if len(word) < 5 or not word.isalpha():
|
|
return word
|
|
for ending in ("ami", "ach", "och", "ovi", "ych", "ymi", "eho", "emu", "iou", "ou", "om", "ov", "ej", "ia", "ie",
|
|
"iu", "im", "y", "a", "e", "i", "u", "o"):
|
|
if word.endswith(ending) and len(word) - len(ending) >= 4:
|
|
return word[: -len(ending)]
|
|
return word
|
|
|
|
|
|
def phrase_pattern(text: str) -> str:
|
|
"""A regex (over normalized text) for the phrase in any Slovak case form."""
|
|
words = _key(text).split()
|
|
if not words:
|
|
return r"(?!)"
|
|
parts = [re.escape(_stem(w)) + (_ENDING if len(w) >= 5 and w.isalpha() else "") for w in words]
|
|
return r"(?<![\w-])" + r"\s+".join(parts) + r"(?![\w-])"
|
|
|
|
|
|
def group_of(text: str) -> list[str]:
|
|
"""The reviewed group the phrase belongs to (any case form), else [text]."""
|
|
key = _key(text)
|
|
for group in GROUPS:
|
|
if any(re.fullmatch(phrase_pattern(item), key) for item in group):
|
|
return group
|
|
return [text]
|
|
|
|
|
|
def base_name(text: str) -> str:
|
|
group = group_of(text)
|
|
return group[0] if len(group) > 1 else text
|
|
|
|
|
|
def variants_pattern(text: str) -> re.Pattern:
|
|
"""One regex matching the topic and its reviewed synonyms in any case form."""
|
|
return re.compile("|".join(phrase_pattern(item) for item in group_of(text)))
|