61 lines
2.2 KiB
Python
61 lines
2.2 KiB
Python
"""CRZP's own vocabulary for questions answered without the model.
|
|
|
|
CRZP matches exact words, so an inflected topic ("rozpoznávaním emócií") must
|
|
be turned into the form the records use. The tags (keywords) of every record
|
|
read before are counted; an inflected phrase is mapped to the most frequent
|
|
tag with the same number of words whose words match it one by one up to
|
|
inflection ("rozpoznávanie emócií", "neurónové siete"). Nothing is invented:
|
|
only phrases CRZP records really carry are used.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
from collections import Counter
|
|
|
|
from tools.crzp.source.parse import fold
|
|
|
|
REFRESH_SECONDS = 3600
|
|
_lock = threading.Lock()
|
|
_state = {"built": 0.0, "phrases": {}}
|
|
|
|
|
|
def _stem_equal(a: str, b: str) -> bool:
|
|
n = max(3, min(len(a), len(b)) - 2)
|
|
return len(a) >= 3 and len(b) >= 3 and a[:n] == b[:n] and abs(len(a) - len(b)) <= 4
|
|
|
|
|
|
def _build() -> dict:
|
|
from tools.crzp.fetch import store
|
|
counts, spelled = Counter(), {}
|
|
for row in store().cached_rows():
|
|
for tag in row.get("keywords", []):
|
|
key = " ".join(fold(tag).split())
|
|
if 1 <= len(key.split()) <= 4 and len(key) <= 60:
|
|
counts[key] += 1
|
|
spelled.setdefault(key, tag.strip())
|
|
by_words: dict = {}
|
|
for key, count in counts.items():
|
|
by_words.setdefault((len(key.split()), key.split()[0][:3]), []).append((count, key))
|
|
return {"by_words": by_words, "spelled": spelled}
|
|
|
|
|
|
def vocabulary() -> dict:
|
|
with _lock:
|
|
if time.monotonic() - _state["built"] > REFRESH_SECONDS:
|
|
_state["phrases"] = _build()
|
|
_state["built"] = time.monotonic()
|
|
return _state["phrases"]
|
|
|
|
|
|
def tag_for(words: list[str]) -> str | None:
|
|
"""The most frequent CRZP tag matching the inflected words one by one, or None."""
|
|
folded = [fold(w) for w in words if w]
|
|
if not folded or len(folded) > 4:
|
|
return None
|
|
table = vocabulary()
|
|
candidates = table.get("by_words", {}).get((len(folded), folded[0][:3]), [])
|
|
best = max(((count, key) for count, key in candidates
|
|
if all(_stem_equal(a, b) for a, b in zip(folded, key.split()))), default=None)
|
|
return table["spelled"][best[1]] if best else None
|