"""Polite HTTP client for the public CRZP OPAC (opac.crzp.sk). The OPAC is a stateful web application: every page carries a form with per-session element ids and actions are AJAX posts of that form. The client replays exactly what the browser sends (no JavaScript engine is needed): search with year and work-type filters, change the page size, go to the next page and export the permalinks (RSS); a record detail is read through its public permalink. Only metadata pages are read; full texts and reviews never. All requests of the process share one throttle: at most one request start per CRZP_MIN_INTERVAL_SECONDS (0.5 s, i.e. 2 requests/s; one CRZP page takes about 1.1 s, so only reads of several year ranges at once overlap). An HTTP 429 slows every request down to CRZP_PENALTY_SECONDS (2 s) for CRZP_PENALTY_DURATION (300 s). """ from __future__ import annotations import html import json import os import re import threading import time import httpx BASE_URL = os.getenv("CRZP_BASE_URL", "https://opac.crzp.sk") USER_AGENT = "Mozilla/5.0 (compatible; zp-agent/1.0; academic thesis-metadata lookup)" PERMALINK = BASE_URL + "/?fn=detailBiblioForm&sid={sid}" # Work type ids of the CRZP search form ("doctoral_thesis" is the rigorous thesis there). CRZP_WORK_TYPES = {"bachelor_thesis", "diploma_thesis", "dissertation", "habilitation_thesis", "doctoral_thesis", "final_internship", "other_monography"} class CrzpUnavailable(RuntimeError): """CRZP did not answer, refused (rate limit) or changed its page structure.""" class _Throttle: """Minimum interval between any two request starts of this process; slower after a 429.""" def __init__(self): self._lock = threading.Lock() self._last = 0.0 self._penalty_until = 0.0 def penalize(self): self._penalty_until = time.monotonic() + float(os.getenv("CRZP_PENALTY_DURATION", "300")) def interval(self) -> float: interval = float(os.getenv("CRZP_MIN_INTERVAL_SECONDS", "0.5")) if time.monotonic() < self._penalty_until: interval = max(interval, float(os.getenv("CRZP_PENALTY_SECONDS", "2.0"))) return interval def wait(self): interval = self.interval() with self._lock: delay = self._last + interval - time.monotonic() if delay > 0: time.sleep(delay) self._last = time.monotonic() THROTTLE = _Throttle() _ASSIGN = re.compile(r'window\.location\.assign\("([^"]+)"\)') def _js_unescape(value: str) -> str: return re.sub(r"\\(.)", lambda m: {"n": "\n", "t": "\t", "r": ""}.get(m.group(1), m.group(1)), value) def _value(pattern: str, page: str) -> str: match = re.search(pattern, page) if not match: raise CrzpUnavailable("page_structure_changed") return match.group(1) def form_fields(page: str) -> dict: """Serialized form like jQuery: inputs (without checkboxes) and textareas.""" fields = {} for tag in re.findall(r"]*>", page): name = re.search(r'name="([^"]+)"', tag) if not name or 'type="checkbox"' in tag: continue value = re.search(r'value="([^"]*)"', tag) fields[name.group(1)] = html.unescape(value.group(1)) if value else "" for name, value in re.findall(r']*name="([^"]+)"[^>]*>(.*?)', page, re.S): fields[name] = html.unescape(value) return fields def action_element(page: str, action: str) -> str | None: """Id of the element whose click triggers the action (first occurrence).""" match = re.search(r"ae\((?:'|')(\w+)(?:'|'),(?:'|')" + re.escape(action) + r"(?:'|')\)", page) return match.group(1) if match else None class CrzpSession: """One OPAC session (cookie + current page). Not shared between threads.""" def __init__(self, timeout: float | None = None): timeout = timeout or float(os.getenv("CRZP_TIMEOUT_SECONDS", "30")) self.http = httpx.Client(base_url=BASE_URL, headers={"User-Agent": USER_AGENT}, follow_redirects=True, timeout=timeout) self.page = "" # last full page: form target and hidden fields self.content = "" # last shown content: a full page or an in-place fragment self.guids = {} self.home_loaded = False # the home page is loaded and no action was done on it yet def load_home(self) -> str: page = self.get("/") self.home_loaded = True return page def close(self): self.http.close() def __enter__(self): return self def __exit__(self, *exc): self.close() def _request(self, method: str, url: str, **kwargs) -> httpx.Response: for attempt in range(3): THROTTLE.wait() try: response = self.http.request(method, url, **kwargs) except httpx.HTTPError as error: if attempt == 2: raise CrzpUnavailable("network_error") from error time.sleep(1.0 + attempt) # one slow or dropped answer is repeated, not reported continue if response.status_code != 429: if response.status_code >= 400: raise CrzpUnavailable(f"http_{response.status_code}") return response THROTTLE.penalize() retry = response.headers.get("Retry-After", "") time.sleep(min(float(retry) if retry.isdigit() else 3.0 * (attempt + 1), 10.0)) raise CrzpUnavailable("rate_limited") def get(self, path: str) -> str: self.page = self.content = self._request("GET", path if path.startswith("/") else "/" + path).text self._guids(self.page) return self.page def _guids(self, text: str) -> None: for name in ("actionGUID", "evntGrpGUID"): match = re.search(name + r'\s*=\s*"(\w+)"', text) if match: self.guids[name] = match.group(1) def act(self, element: str, action: str, extra: dict | None = None) -> str: """AJAX action of the current page. Returns the next page (followed redirect) or the fragment that replaces part of the page in place.""" target = html.unescape(_value(r']*id="submitform"[^>]*action="([^"]+)"', self.page)) fields = {**form_fields(self.page), **(form_fields(self.content) if self.content is not self.page else {})} fields.update(extra or {}) if set(self.guids) != {"actionGUID", "evntGrpGUID"}: raise CrzpUnavailable("page_structure_changed") fields.update(fo=element, aa=action, dwcall="ajax", iact="true", egrp="false", acGUID=self.guids["actionGUID"], acEvntGrpGUID=self.guids["evntGrpGUID"]) script = self._request("POST", "/" + target.lstrip("/"), data=fields, headers={"X-Requested-With": "XMLHttpRequest"}).text location = _ASSIGN.search(script) if location: return self.get(location.group(1)) if 'id="submitform"' in script: # the whole new page self.page = self.content = script self._guids(script) return script self._guids(script) fragments = re.findall(r"\.replaceWith\('((?:[^'\\]|\\.)*)'\)", script, re.S) if fragments: self.content = "".join(_js_unescape(f) for f in fragments) return self.content return script def search(self, text: str, *, years: tuple[int, int] | None = None, work_types: list[str] | None = None) -> str: """Search form of the home page: expression, optional range of submission years and work types (both filtered by CRZP itself). Returns the first result page.""" page = self.page if self.home_loaded else self.get("/") self.home_loaded = False expression = re.search(r'id="divEbExpression">.*?]*name="(\w+)"', page, re.S) year_from = re.search(r'name="(\w+)"\s+placeholder="Od\.\.\."', page) year_to = re.search(r'name="(\w+)"\s+placeholder="Do\.\.\."', page) types = re.search(r']*name="(\w+)">\["bachelor_thesis', page) button = action_element(page, "anDoSearchClickAjax") if not (expression and year_from and year_to and types and button): raise CrzpUnavailable("page_structure_changed") extra = {expression.group(1): text} if years: extra[year_from.group(1)], extra[year_to.group(1)] = str(years[0]), str(years[1]) if work_types: unknown = set(work_types) - CRZP_WORK_TYPES if unknown: raise ValueError("unknown CRZP work type") extra[types.group(1)] = json.dumps(sorted(work_types)) page = self.act(button, "anDoSearchClickAjax", extra) if "Počet nájdených záznamov" not in page and "Nenašli sa" not in page and "items-list" not in page: raise CrzpUnavailable("page_structure_changed") return page def set_page_size(self) -> str: """Switch the result list to 100 records per page (option "2").""" match = re.search(r'"storageSelector":"textarea\[name=(\w+)\]"[^<]*?"rechmItemsPerPageChangeAjax"', self.content) if not match: raise CrzpUnavailable("page_structure_changed") return self.act(match.group(1), "rechmItemsPerPageChangeAjax", {match.group(1): '["2"]'}) def next_page(self) -> str | None: """Next result page, or None on the last one.""" match = re.search(r"ae\('(\w+)','(navigator:navBiblio_\d+)'\);\" title=\"\+1\"", self.content) return self.act(match.group(1), match.group(2)) if match else None def show_all(self, element: str) -> list[dict]: """Every value of a facet ("Rok odovzdania", "Školiteľ"): [{label, count}] (one request).""" script = self.act(element, "showMoreResultsNewAjax") if "[" not in script: # right after another expansion CRZP first only renews the action GUID script = self.act(element, "showMoreResultsNewAjax") start = script.find("[") try: items = json.loads(script[start:script.rfind("]") + 1]) if start >= 0 else None except ValueError: items = None if not isinstance(items, list) or not all(isinstance(i, dict) and "label" in i and "count" in i for i in items): raise CrzpUnavailable("page_structure_changed") return [{"label": str(i["label"]), "count": int(i["count"])} for i in items] def rss_links(self) -> str: """RSS export of the current search: titles and permalinks (CRZP caps it at 200).""" element = action_element(self.content, "makeRSSChannelAjax") or action_element(self.page, "makeRSSChannelAjax") if element is None: raise CrzpUnavailable("page_structure_changed") script = self.act(element, "makeRSSChannelAjax") url = _value(r'window\.open\("([^"]+)"', script) return self._request("GET", "/" + url.lstrip("/")).text def detail_by_sid(self, sid: str) -> str: if not re.fullmatch(r"[0-9A-F]{16,64}", sid): raise ValueError("invalid CRZP record id") return self.get(f"/?fn=detailBiblioForm&sid={sid}")