36 lines
1.9 KiB
Python
36 lines
1.9 KiB
Python
"""Curated equivalents, never arbitrary related tasks or inferred relations."""
|
|
import re
|
|
from tools.answers.context_policy import normalized
|
|
|
|
|
|
def concept_key(text: str) -> str:
|
|
value = normalized(text)
|
|
equivalents = {
|
|
'rag': {'rag', 'retrieval augmented generation', 'generovanie s podporou vyhladavania'},
|
|
'ner': {'ner', 'named entity recognition', 'rozpoznavanie pomenovanych entit'},
|
|
'qa': {'qa', 'question answering', 'odpovedanie na otazky'},
|
|
'knowledge graph': {'knowledge graph', 'znalostny graf', 'znalostne grafy'},
|
|
'chatbot': {'chatbot', 'chatboty', 'chatbota'},
|
|
}
|
|
return next((key for key, aliases in equivalents.items() if value in aliases), value)
|
|
|
|
|
|
GENERIC_CONCEPTS = {'python', 'nlp', 'system', 'model', 'student', 'praca', 'projekt', 'git',
|
|
'docker', 'openwebui', 'rozhranie', 'webhook', 'generativny model', 'vyhladavat', 'data', 'text',
|
|
'llm', 'language model', 'jazykovy model', 'neural network', 'neuronova siet',
|
|
'machine learning', 'strojove ucenie', 'artificial intelligence', 'umela inteligencia',
|
|
'api', 'web', 'database', 'application', 'software', 'studium'}
|
|
|
|
|
|
def substantive_concept(value):
|
|
"""Reading/writing infrastructure is not a research-topic similarity."""
|
|
words=normalized(value).split()
|
|
return bool(words) and normalized(value) not in GENERIC_CONCEPTS and not all(word in GENERIC_CONCEPTS or re.fullmatch(
|
|
r'clank\w*|poznam\w*|literatur\w*|resers\w*|report\w*|text\w*|kod\w*|skript\w*|'
|
|
r'zdroj\w*|prehlad\w*|teori\w*|konzult\w*|stretnut\w*|prezent\w*|'
|
|
r'implementac\w*|article\w*|paper\w*|note\w*|research|literature|code|script\w*|'
|
|
r'writing|reading|source\w*|meeting\w*|tutorial\w*|experiment\w*|metod\w*|method\w*|'
|
|
r'theor\w*|bibliograph\w*|citac\w*|citation\w*|vysled\w*|uloh\w*|zadani\w*|priprava|citanie|pisanie|'
|
|
r'o|na|a|s|z|pre|of|on|and|for|the|in|to',word)
|
|
for word in words)
|