256 lines
11 KiB
Python
256 lines
11 KiB
Python
"""Cached CRZP search and record detail (source + storage).
|
|
|
|
A search may carry a range of submission years and work types; CRZP applies
|
|
both itself, so the total, the facets and the read rows already respect them.
|
|
Rows are read in pages of 100 up to `depth` (LINKED_ROWS by default, up to
|
|
DEEP_ROWS for counts and summaries). The RSS export, which carries the
|
|
permalink ids, is capped by CRZP at LINKED_ROWS and does not follow a changed
|
|
sort order, so only the first LINKED_ROWS rows (same order; a title mismatch
|
|
leaves the row without id instead of guessing) get a permalink; counts and
|
|
summaries do not need one.
|
|
|
|
While the model reads the question, `prepare()` loads the CRZP home page in a
|
|
background session, so the first search of the question saves that request.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import threading
|
|
import time
|
|
|
|
from tools.crzp.source.client import CrzpSession, CrzpUnavailable, PERMALINK
|
|
from tools.crzp.source.parse import facet_actions, fold, parse_detail, parse_results, parse_rows, parse_rss
|
|
from tools.crzp.storage.cache import CrzpStore
|
|
|
|
LINKED_ROWS = 200 # the RSS export of CRZP is capped at 200 items
|
|
DEEP_ROWS = int(os.getenv("CRZP_DEEP_ROWS", "3000")) # summaries: read in year ranges, several at once
|
|
PARALLEL = int(os.getenv("CRZP_PARALLEL", "3")) # year ranges read at the same time (shared throttle)
|
|
EXPANDED = ("Rok odovzdania", "Školiteľ") # facets read in full when a result exceeds one page
|
|
MAX_ROWS = LINKED_ROWS
|
|
PREPARED_SECONDS = 120
|
|
_prepared: list = [] # [(created, session with the home page loaded)]
|
|
_prepared_lock = threading.Lock()
|
|
_store = None
|
|
_store_lock = threading.Lock()
|
|
_status = {"last_success": None, "last_failure": None, "last_reason": None,
|
|
"canary_ok": None, "canary_at": None, "canary_reason": None}
|
|
|
|
|
|
def record_outcome(error: Exception | None) -> None:
|
|
if error is None:
|
|
_status.update(last_success=time.time())
|
|
else:
|
|
_status.update(last_failure=time.time(), last_reason=str(error))
|
|
|
|
|
|
def record_canary(ok: bool, reason: str | None = None) -> None:
|
|
_status.update(canary_ok=ok, canary_at=time.time(), canary_reason=reason)
|
|
|
|
|
|
def readiness() -> dict:
|
|
"""Health of the CRZP tool without contacting CRZP (no traffic on health polls).
|
|
"degraded": the last request failed or the daily check of the page structure failed."""
|
|
failed_last = bool(_status["last_failure"]) and (_status["last_success"] or 0) < _status["last_failure"]
|
|
canary_failed = _status["canary_ok"] is False
|
|
reason = _status["canary_reason"] if canary_failed else _status["last_reason"] if failed_last else None
|
|
return {"ready": True, "status": "degraded" if failed_last or canary_failed else "ready", "reason": reason,
|
|
"index_version": None, "cache": store().stats(),
|
|
"structure_check": {"ok": _status["canary_ok"], "checked_at": _status["canary_at"]}}
|
|
|
|
|
|
def store() -> CrzpStore:
|
|
global _store
|
|
with _store_lock:
|
|
if _store is None:
|
|
_store = CrzpStore()
|
|
return _store
|
|
|
|
|
|
def cache_key(text: str, years: tuple[int, int] | None, work_types: list[str] | None) -> str:
|
|
key = " ".join(fold(text).split())
|
|
if years:
|
|
key += f" |years={years[0]}-{years[1]}"
|
|
if work_types:
|
|
key += " |types=" + ",".join(sorted(work_types))
|
|
return key
|
|
|
|
|
|
def _attach_links(rows: list[dict], links: list[dict]) -> None:
|
|
for row, link in zip(rows, links):
|
|
if link["sid"] and fold(link["title"]) == fold(row["title"]):
|
|
row["sid"] = link["sid"]
|
|
row["url"] = PERMALINK.format(sid=link["sid"])
|
|
|
|
|
|
def prepare() -> None:
|
|
"""Load the home page in the background (one request) for the next search."""
|
|
if os.getenv("CRZP_PREPARE", "1") == "0":
|
|
return
|
|
def run():
|
|
try:
|
|
session = CrzpSession()
|
|
session.load_home()
|
|
except Exception:
|
|
return
|
|
with _prepared_lock:
|
|
_prepared.append((time.monotonic(), session))
|
|
while len(_prepared) > 1:
|
|
_prepared.pop(0)[1].close()
|
|
with _prepared_lock:
|
|
if _prepared and time.monotonic() - _prepared[-1][0] < PREPARED_SECONDS:
|
|
return
|
|
threading.Thread(target=run, name="crzp-prepare", daemon=True).start()
|
|
|
|
|
|
def _session():
|
|
with _prepared_lock:
|
|
while _prepared:
|
|
created, session = _prepared.pop()
|
|
if time.monotonic() - created < PREPARED_SECONDS:
|
|
return session
|
|
session.close()
|
|
return CrzpSession()
|
|
|
|
|
|
def _enough(hit: dict, links: bool, depth: int) -> bool:
|
|
return ((hit.get("links", True) or not links)
|
|
and (hit["complete"] or len(hit["rows"]) >= min(hit["total"], depth)))
|
|
|
|
|
|
def search(text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None,
|
|
links: bool = True, depth: int = LINKED_ROWS, session_factory=None,
|
|
cache: CrzpStore | None = None) -> dict:
|
|
"""{query, years, work_types, total, facets, rows, complete, links, cached}.
|
|
links=False skips the RSS export (two requests) when permalinks are not shown;
|
|
depth is the number of rows to read (DEEP_ROWS for counts and summaries)."""
|
|
cache = cache or store()
|
|
key = cache_key(text, years, work_types)
|
|
hit = cache.get_search(key)
|
|
if hit is not None and _enough(hit, links, depth):
|
|
if len(hit["rows"]) > depth: # a deeper cached read: the same rows a shallow read would give
|
|
hit = {**hit, "rows": hit["rows"][:depth], "complete": False}
|
|
return {**hit, "cached": True}
|
|
try:
|
|
result = _search_live(text, years, work_types, links or bool(hit and hit.get("links")),
|
|
session_factory or _session, depth)
|
|
except CrzpUnavailable as error:
|
|
record_outcome(error)
|
|
raise
|
|
record_outcome(None)
|
|
cache.put_search(key, result)
|
|
return {**result, "cached": False}
|
|
|
|
|
|
def _expanded(session, first: dict, actions: dict) -> dict:
|
|
"""All values of the year and supervisor facets (two requests), exact over the whole result."""
|
|
full = {}
|
|
for title in EXPANDED:
|
|
if title in actions and len(first["facets"].get(title, [])) >= 10: # CRZP shows the top 10 only
|
|
full[title] = session.show_all(actions[title])
|
|
elif title in first["facets"]:
|
|
full[title] = first["facets"][title]
|
|
return full
|
|
|
|
|
|
def _search_live(text, years, work_types, links, session_factory, depth=LINKED_ROWS) -> dict:
|
|
with session_factory() as session:
|
|
first = parse_results(session.search(text, years=years, work_types=work_types))
|
|
actions = facet_actions(getattr(session, "content", "") or "") # ids from the first result page
|
|
rows = first["rows"]
|
|
if first["total"] > len(rows) and depth > len(rows): # up to 20 results: one page, no paging requests
|
|
rows = parse_rows(session.set_page_size())
|
|
while len(rows) < min(first["total"], depth):
|
|
page = session.next_page()
|
|
if page is None:
|
|
break
|
|
more = parse_rows(page)
|
|
if not more:
|
|
break
|
|
rows += more
|
|
rows = rows[:depth]
|
|
if rows and links:
|
|
_attach_links(rows[:LINKED_ROWS], parse_rss(session.rss_links()))
|
|
if first["total"] > len(first["rows"]): # last: after an expansion CRZP pages no more rows
|
|
first["facets"] = {**first["facets"], **_expanded(session, first, actions)}
|
|
return {"query": text, "years": list(years) if years else None, "work_types": work_types or None,
|
|
"total": first["total"], "facets": first["facets"], "rows": rows, "complete": len(rows) >= first["total"],
|
|
"links": links}
|
|
|
|
|
|
def detail(sid: str, *, session_factory=CrzpSession, cache: CrzpStore | None = None) -> dict:
|
|
cache = cache or store()
|
|
hit = cache.get_detail(sid)
|
|
if hit is not None:
|
|
return hit
|
|
try:
|
|
with session_factory() as session:
|
|
record = parse_detail(session.detail_by_sid(sid))
|
|
except CrzpUnavailable as error:
|
|
record_outcome(error)
|
|
raise
|
|
record_outcome(None)
|
|
record["url"] = PERMALINK.format(sid=record["sid"])
|
|
cache.put_detail(sid, record)
|
|
return record
|
|
|
|
|
|
_chunk_pool = {"pool": None}
|
|
|
|
|
|
def _pool():
|
|
"""One shared pool for year-range reads (no new threads per search)."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
with _store_lock:
|
|
if _chunk_pool["pool"] is None:
|
|
_chunk_pool["pool"] = ThreadPoolExecutor(max_workers=max(1, PARALLEL), thread_name_prefix="crzp-range")
|
|
return _chunk_pool["pool"]
|
|
|
|
|
|
def year_counts(result: dict) -> dict[int, int]:
|
|
return {int(f["label"]): f["count"] for f in result["facets"].get("Rok odovzdania", [])
|
|
if str(f["label"]).isdigit()}
|
|
|
|
|
|
def plan(counts: dict[int, int], years, cap: int, limit: int) -> list[tuple[int, int]]:
|
|
"""Year ranges, newest first, each with at most `cap` records (one year may exceed it),
|
|
until `limit` records are planned."""
|
|
low, high = years or (min(counts), max(counts))
|
|
chunks, current, size, planned = [], [], 0, 0
|
|
for year, count in sorted(((y, n) for y, n in counts.items() if low <= y <= high), reverse=True):
|
|
if planned >= limit:
|
|
break
|
|
if current and size + count > cap:
|
|
chunks.append((min(current), max(current)))
|
|
current, size = [], 0
|
|
current.append(year)
|
|
size += count
|
|
planned += count
|
|
if current:
|
|
chunks.append((min(current), max(current)))
|
|
return chunks
|
|
|
|
|
|
def search_deep(text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None,
|
|
links: bool = False, limit: int = DEEP_ROWS, cache: CrzpStore | None = None) -> dict:
|
|
"""Up to `limit` rows of a result larger than one read, read in year ranges
|
|
(the year facet tells how many records each year has), PARALLEL ranges at once.
|
|
With links, every range has at most LINKED_ROWS records, so its RSS export
|
|
gives a permalink to every row; without links the ranges are larger."""
|
|
head = search(text, years=years, work_types=work_types, links=links,
|
|
depth=LINKED_ROWS if links else 20, cache=cache)
|
|
if head["complete"]:
|
|
return head
|
|
counts = year_counts(head)
|
|
if not counts:
|
|
return search(text, years=years, work_types=work_types, links=links, depth=limit, cache=cache)
|
|
wanted = min(head["total"], limit)
|
|
cap = LINKED_ROWS if links else max(LINKED_ROWS, -(-wanted // PARALLEL))
|
|
chunks = plan(counts, years, cap, limit)
|
|
parts = list(_pool().map(lambda c: search(text, years=c, work_types=work_types, links=links,
|
|
depth=LINKED_ROWS if links else limit, cache=cache), chunks))
|
|
rows = [r for part in parts for r in part["rows"]][:limit]
|
|
read_total = sum(part["total"] for part in parts)
|
|
return {**head, "rows": rows, "complete": len(rows) >= head["total"], "chunks": chunks,
|
|
"links": links, "cached": all(part.get("cached") for part in parts),
|
|
"chunks_complete": all(part["complete"] for part in parts) and read_total >= head["total"]}
|