180 lines
7.2 KiB
Python
180 lines
7.2 KiB
Python
"""Cached CRZP search and record detail (source + storage).
|
|
|
|
A search may carry a range of submission years and work types; CRZP applies
|
|
both itself, so the total, the facets and the read rows already respect them.
|
|
Rows are read in pages of 100 up to `depth` (LINKED_ROWS by default, up to
|
|
DEEP_ROWS for counts and summaries). The RSS export, which carries the
|
|
permalink ids, is capped by CRZP at LINKED_ROWS and does not follow a changed
|
|
sort order, so only the first LINKED_ROWS rows (same order; a title mismatch
|
|
leaves the row without id instead of guessing) get a permalink; counts and
|
|
summaries do not need one.
|
|
|
|
While the model reads the question, `prepare()` loads the CRZP home page in a
|
|
background session, so the first search of the question saves that request.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import threading
|
|
import time
|
|
|
|
from tools.crzp.source.client import CrzpSession, CrzpUnavailable, PERMALINK
|
|
from tools.crzp.source.parse import fold, parse_detail, parse_results, parse_rows, parse_rss
|
|
from tools.crzp.storage.cache import CrzpStore
|
|
|
|
LINKED_ROWS = 200 # the RSS export of CRZP is capped at 200 items
|
|
DEEP_ROWS = int(os.getenv("CRZP_DEEP_ROWS", "1000")) # counts and summaries: up to 10 pages of 100
|
|
MAX_ROWS = LINKED_ROWS
|
|
PREPARED_SECONDS = 120
|
|
_prepared: list = [] # [(created, session with the home page loaded)]
|
|
_prepared_lock = threading.Lock()
|
|
_store = None
|
|
_store_lock = threading.Lock()
|
|
_status = {"last_success": None, "last_failure": None, "last_reason": None,
|
|
"canary_ok": None, "canary_at": None, "canary_reason": None}
|
|
|
|
|
|
def record_outcome(error: Exception | None) -> None:
|
|
if error is None:
|
|
_status.update(last_success=time.time())
|
|
else:
|
|
_status.update(last_failure=time.time(), last_reason=str(error))
|
|
|
|
|
|
def record_canary(ok: bool, reason: str | None = None) -> None:
|
|
_status.update(canary_ok=ok, canary_at=time.time(), canary_reason=reason)
|
|
|
|
|
|
def readiness() -> dict:
|
|
"""Health of the CRZP tool without contacting CRZP (no traffic on health polls).
|
|
"degraded": the last request failed or the daily check of the page structure failed."""
|
|
failed_last = bool(_status["last_failure"]) and (_status["last_success"] or 0) < _status["last_failure"]
|
|
canary_failed = _status["canary_ok"] is False
|
|
reason = _status["canary_reason"] if canary_failed else _status["last_reason"] if failed_last else None
|
|
return {"ready": True, "status": "degraded" if failed_last or canary_failed else "ready", "reason": reason,
|
|
"index_version": None, "cache": store().stats(),
|
|
"structure_check": {"ok": _status["canary_ok"], "checked_at": _status["canary_at"]}}
|
|
|
|
|
|
def store() -> CrzpStore:
|
|
global _store
|
|
with _store_lock:
|
|
if _store is None:
|
|
_store = CrzpStore()
|
|
return _store
|
|
|
|
|
|
def cache_key(text: str, years: tuple[int, int] | None, work_types: list[str] | None) -> str:
|
|
key = " ".join(fold(text).split())
|
|
if years:
|
|
key += f" |years={years[0]}-{years[1]}"
|
|
if work_types:
|
|
key += " |types=" + ",".join(sorted(work_types))
|
|
return key
|
|
|
|
|
|
def _attach_links(rows: list[dict], links: list[dict]) -> None:
|
|
for row, link in zip(rows, links):
|
|
if link["sid"] and fold(link["title"]) == fold(row["title"]):
|
|
row["sid"] = link["sid"]
|
|
row["url"] = PERMALINK.format(sid=link["sid"])
|
|
|
|
|
|
def prepare() -> None:
|
|
"""Load the home page in the background (one request) for the next search."""
|
|
if os.getenv("CRZP_PREPARE", "1") == "0":
|
|
return
|
|
def run():
|
|
try:
|
|
session = CrzpSession()
|
|
session.load_home()
|
|
except Exception:
|
|
return
|
|
with _prepared_lock:
|
|
_prepared.append((time.monotonic(), session))
|
|
while len(_prepared) > 1:
|
|
_prepared.pop(0)[1].close()
|
|
with _prepared_lock:
|
|
if _prepared and time.monotonic() - _prepared[-1][0] < PREPARED_SECONDS:
|
|
return
|
|
threading.Thread(target=run, name="crzp-prepare", daemon=True).start()
|
|
|
|
|
|
def _session():
|
|
with _prepared_lock:
|
|
while _prepared:
|
|
created, session = _prepared.pop()
|
|
if time.monotonic() - created < PREPARED_SECONDS:
|
|
return session
|
|
session.close()
|
|
return CrzpSession()
|
|
|
|
|
|
def _enough(hit: dict, links: bool, depth: int) -> bool:
|
|
return ((hit.get("links", True) or not links)
|
|
and (hit["complete"] or len(hit["rows"]) >= min(hit["total"], depth)))
|
|
|
|
|
|
def search(text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None,
|
|
links: bool = True, depth: int = LINKED_ROWS, session_factory=None,
|
|
cache: CrzpStore | None = None) -> dict:
|
|
"""{query, years, work_types, total, facets, rows, complete, links, cached}.
|
|
links=False skips the RSS export (two requests) when permalinks are not shown;
|
|
depth is the number of rows to read (DEEP_ROWS for counts and summaries)."""
|
|
cache = cache or store()
|
|
key = cache_key(text, years, work_types)
|
|
hit = cache.get_search(key)
|
|
if hit is not None and _enough(hit, links, depth):
|
|
if len(hit["rows"]) > depth: # a deeper cached read: the same rows a shallow read would give
|
|
hit = {**hit, "rows": hit["rows"][:depth], "complete": False}
|
|
return {**hit, "cached": True}
|
|
try:
|
|
result = _search_live(text, years, work_types, links or bool(hit and hit.get("links")),
|
|
session_factory or _session, depth)
|
|
except CrzpUnavailable as error:
|
|
record_outcome(error)
|
|
raise
|
|
record_outcome(None)
|
|
cache.put_search(key, result)
|
|
return {**result, "cached": False}
|
|
|
|
|
|
def _search_live(text, years, work_types, links, session_factory, depth=LINKED_ROWS) -> dict:
|
|
depth = max(depth, LINKED_ROWS)
|
|
with session_factory() as session:
|
|
first = parse_results(session.search(text, years=years, work_types=work_types))
|
|
rows = first["rows"]
|
|
if first["total"] > len(rows): # up to 20 results: one page, no paging requests
|
|
rows = parse_rows(session.set_page_size())
|
|
while len(rows) < min(first["total"], depth):
|
|
page = session.next_page()
|
|
if page is None:
|
|
break
|
|
more = parse_rows(page)
|
|
if not more:
|
|
break
|
|
rows += more
|
|
rows = rows[:depth]
|
|
if rows and links:
|
|
_attach_links(rows[:LINKED_ROWS], parse_rss(session.rss_links()))
|
|
return {"query": text, "years": list(years) if years else None, "work_types": work_types or None,
|
|
"total": first["total"], "facets": first["facets"], "rows": rows, "complete": len(rows) >= first["total"],
|
|
"links": links}
|
|
|
|
|
|
def detail(sid: str, *, session_factory=CrzpSession, cache: CrzpStore | None = None) -> dict:
|
|
cache = cache or store()
|
|
hit = cache.get_detail(sid)
|
|
if hit is not None:
|
|
return hit
|
|
try:
|
|
with session_factory() as session:
|
|
record = parse_detail(session.detail_by_sid(sid))
|
|
except CrzpUnavailable as error:
|
|
record_outcome(error)
|
|
raise
|
|
record_outcome(None)
|
|
record["url"] = PERMALINK.format(sid=record["sid"])
|
|
cache.put_detail(sid, record)
|
|
return record
|