dp-zp-agent/tools/crzp/source/client.py

"""Polite HTTP client for the public CRZP OPAC (opac.crzp.sk).

The OPAC is a stateful web application: every page carries a form with
per-session element ids and actions are AJAX posts of that form. The client
replays exactly what the browser sends (no JavaScript engine is needed):
search with year and work-type filters, change the page size, go to the next page and export
the permalinks (RSS); a record detail is read through its public permalink. Only metadata pages are read; full texts and reviews never.

All requests of the process share one throttle: at most one request start per
CRZP_MIN_INTERVAL_SECONDS (0.5 s, i.e. 2 requests/s; one CRZP page takes about
1.1 s, so only reads of several year ranges at once overlap). An HTTP 429 slows
every request down to CRZP_PENALTY_SECONDS (2 s) for CRZP_PENALTY_DURATION (300 s).
"""
from __future__ import annotations

import html
import json
import os
import re
import threading
import time

import httpx

BASE_URL = os.getenv("CRZP_BASE_URL", "https://opac.crzp.sk")
USER_AGENT = "Mozilla/5.0 (compatible; zp-agent/1.0; academic thesis-metadata lookup)"
PERMALINK = BASE_URL + "/?fn=detailBiblioForm&sid={sid}"
# Work type ids of the CRZP search form ("doctoral_thesis" is the rigorous thesis there).
CRZP_WORK_TYPES = {"bachelor_thesis", "diploma_thesis", "dissertation", "habilitation_thesis", "doctoral_thesis",
                   "final_internship", "other_monography"}


class CrzpUnavailable(RuntimeError):
    """CRZP did not answer, refused (rate limit) or changed its page structure."""


class _Throttle:
    """Minimum interval between any two request starts of this process; slower after a 429."""

    def __init__(self):
        self._lock = threading.Lock()
        self._last = 0.0
        self._penalty_until = 0.0

    def penalize(self):
        self._penalty_until = time.monotonic() + float(os.getenv("CRZP_PENALTY_DURATION", "300"))

    def interval(self) -> float:
        interval = float(os.getenv("CRZP_MIN_INTERVAL_SECONDS", "0.5"))
        if time.monotonic() < self._penalty_until:
            interval = max(interval, float(os.getenv("CRZP_PENALTY_SECONDS", "2.0")))
        return interval

    def wait(self):
        interval = self.interval()
        with self._lock:
            delay = self._last + interval - time.monotonic()
            if delay > 0:
                time.sleep(delay)
            self._last = time.monotonic()


THROTTLE = _Throttle()
_ASSIGN = re.compile(r'window\.location\.assign\("([^"]+)"\)')


def _js_unescape(value: str) -> str:
    return re.sub(r"\\(.)", lambda m: {"n": "\n", "t": "\t", "r": ""}.get(m.group(1), m.group(1)), value)


def _value(pattern: str, page: str) -> str:
    match = re.search(pattern, page)
    if not match:
        raise CrzpUnavailable("page_structure_changed")
    return match.group(1)


def form_fields(page: str) -> dict:
    """Serialized form like jQuery: inputs (without checkboxes) and textareas."""
    fields = {}
    for tag in re.findall(r"<input[^>]*>", page):
        name = re.search(r'name="([^"]+)"', tag)
        if not name or 'type="checkbox"' in tag:
            continue
        value = re.search(r'value="([^"]*)"', tag)
        fields[name.group(1)] = html.unescape(value.group(1)) if value else ""
    for name, value in re.findall(r'<textarea[^>]*name="([^"]+)"[^>]*>(.*?)</textarea>', page, re.S):
        fields[name] = html.unescape(value)
    return fields


def action_element(page: str, action: str) -> str | None:
    """Id of the element whose click triggers the action (first occurrence)."""
    match = re.search(r"ae\((?:'|&#39;)(\w+)(?:'|&#39;),(?:'|&#39;)" + re.escape(action) + r"(?:'|&#39;)\)", page)
    return match.group(1) if match else None


class CrzpSession:
    """One OPAC session (cookie + current page). Not shared between threads."""

    def __init__(self, timeout: float | None = None):
        timeout = timeout or float(os.getenv("CRZP_TIMEOUT_SECONDS", "30"))
        self.http = httpx.Client(base_url=BASE_URL, headers={"User-Agent": USER_AGENT},
                                 follow_redirects=True, timeout=timeout)
        self.page = ""     # last full page: form target and hidden fields
        self.content = ""  # last shown content: a full page or an in-place fragment
        self.guids = {}
        self.home_loaded = False  # the home page is loaded and no action was done on it yet

    def load_home(self) -> str:
        page = self.get("/")
        self.home_loaded = True
        return page

    def close(self):
        self.http.close()

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        self.close()

    def _request(self, method: str, url: str, **kwargs) -> httpx.Response:
        for attempt in range(3):
            THROTTLE.wait()
            try:
                response = self.http.request(method, url, **kwargs)
            except httpx.HTTPError as error:
                if attempt == 2:
                    raise CrzpUnavailable("network_error") from error
                time.sleep(1.0 + attempt)  # one slow or dropped answer is repeated, not reported
                continue
            if response.status_code != 429:
                if response.status_code >= 400:
                    raise CrzpUnavailable(f"http_{response.status_code}")
                return response
            THROTTLE.penalize()
            retry = response.headers.get("Retry-After", "")
            time.sleep(min(float(retry) if retry.isdigit() else 3.0 * (attempt + 1), 10.0))
        raise CrzpUnavailable("rate_limited")

    def get(self, path: str) -> str:
        self.page = self.content = self._request("GET", path if path.startswith("/") else "/" + path).text
        self._guids(self.page)
        return self.page

    def _guids(self, text: str) -> None:
        for name in ("actionGUID", "evntGrpGUID"):
            match = re.search(name + r'\s*=\s*"(\w+)"', text)
            if match:
                self.guids[name] = match.group(1)

    def act(self, element: str, action: str, extra: dict | None = None) -> str:
        """AJAX action of the current page. Returns the next page (followed redirect)
        or the fragment that replaces part of the page in place."""
        target = html.unescape(_value(r'<form[^>]*id="submitform"[^>]*action="([^"]+)"', self.page))
        fields = {**form_fields(self.page), **(form_fields(self.content) if self.content is not self.page else {})}
        fields.update(extra or {})
        if set(self.guids) != {"actionGUID", "evntGrpGUID"}:
            raise CrzpUnavailable("page_structure_changed")
        fields.update(fo=element, aa=action, dwcall="ajax", iact="true", egrp="false",
                      acGUID=self.guids["actionGUID"], acEvntGrpGUID=self.guids["evntGrpGUID"])
        script = self._request("POST", "/" + target.lstrip("/"), data=fields,
                               headers={"X-Requested-With": "XMLHttpRequest"}).text
        location = _ASSIGN.search(script)
        if location:
            return self.get(location.group(1))
        if 'id="submitform"' in script:  # the whole new page
            self.page = self.content = script
            self._guids(script)
            return script
        self._guids(script)
        fragments = re.findall(r"\.replaceWith\('((?:[^'\\]|\\.)*)'\)", script, re.S)
        if fragments:
            self.content = "".join(_js_unescape(f) for f in fragments)
            return self.content
        return script

    def search(self, text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None) -> str:
        """Search form of the home page: expression, optional range of submission
        years and work types (both filtered by CRZP itself). Returns the first
        result page."""
        page = self.page if self.home_loaded else self.get("/")
        self.home_loaded = False
        expression = re.search(r'id="divEbExpression">.*?<input[^>]*name="(\w+)"', page, re.S)
        year_from = re.search(r'name="(\w+)"\s+placeholder="Od\.\.\."', page)
        year_to = re.search(r'name="(\w+)"\s+placeholder="Do\.\.\."', page)
        types = re.search(r'<textarea[^>]*name="(\w+)">\[&quot;bachelor_thesis', page)
        button = action_element(page, "anDoSearchClickAjax")
        if not (expression and year_from and year_to and types and button):
            raise CrzpUnavailable("page_structure_changed")
        extra = {expression.group(1): text}
        if years:
            extra[year_from.group(1)], extra[year_to.group(1)] = str(years[0]), str(years[1])
        if work_types:
            unknown = set(work_types) - CRZP_WORK_TYPES
            if unknown:
                raise ValueError("unknown CRZP work type")
            extra[types.group(1)] = json.dumps(sorted(work_types))
        page = self.act(button, "anDoSearchClickAjax", extra)
        if "Počet nájdených záznamov" not in page and "Nenašli sa" not in page and "items-list" not in page:
            raise CrzpUnavailable("page_structure_changed")
        return page

    def set_page_size(self) -> str:
        """Switch the result list to 100 records per page (option "2")."""
        match = re.search(r'"storageSelector":"textarea\[name=(\w+)\]"[^<]*?"rechmItemsPerPageChangeAjax"', self.content)
        if not match:
            raise CrzpUnavailable("page_structure_changed")
        return self.act(match.group(1), "rechmItemsPerPageChangeAjax", {match.group(1): '["2"]'})

    def next_page(self) -> str | None:
        """Next result page, or None on the last one."""
        match = re.search(r"ae\(&#39;(\w+)&#39;,&#39;(navigator:navBiblio_\d+)&#39;\);\" title=\"\+1\"", self.content)
        return self.act(match.group(1), match.group(2)) if match else None

    def show_all(self, element: str) -> list[dict]:
        """Every value of a facet ("Rok odovzdania", "Školiteľ"): [{label, count}] (one request)."""
        script = self.act(element, "showMoreResultsNewAjax")
        if "[" not in script:  # right after another expansion CRZP first only renews the action GUID
            script = self.act(element, "showMoreResultsNewAjax")
        start = script.find("[")
        try:
            items = json.loads(script[start:script.rfind("]") + 1]) if start >= 0 else None
        except ValueError:
            items = None
        if not isinstance(items, list) or not all(isinstance(i, dict) and "label" in i and "count" in i for i in items):
            raise CrzpUnavailable("page_structure_changed")
        return [{"label": str(i["label"]), "count": int(i["count"])} for i in items]

    def rss_links(self) -> str:
        """RSS export of the current search: titles and permalinks (CRZP caps it at 200)."""
        element = action_element(self.content, "makeRSSChannelAjax") or action_element(self.page, "makeRSSChannelAjax")
        if element is None:
            raise CrzpUnavailable("page_structure_changed")
        script = self.act(element, "makeRSSChannelAjax")
        url = _value(r'window\.open\("([^"]+)"', script)
        return self._request("GET", "/" + url.lstrip("/")).text

    def detail_by_sid(self, sid: str) -> str:
        if not re.fullmatch(r"[0-9A-F]{16,64}", sid):
            raise ValueError("invalid CRZP record id")
        return self.get(f"/?fn=detailBiblioForm&sid={sid}")