202 lines
8.8 KiB
Python
202 lines
8.8 KiB
Python
"""Parse CRZP OPAC pages into plain metadata records.
|
|
|
|
A result row already carries the work type, title, author, supervisor(s),
|
|
opponent(s), keywords (CRZP tags), school and year. A detail page adds the
|
|
abstracts, page count, language, field of study and the permalink id (sid).
|
|
Work type ids are those of the CRZP search form (rigorózna = doctoral_thesis).
|
|
A page that does not have the expected structure raises an error; values are
|
|
never guessed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
|
|
from tools.crzp.source.client import CrzpUnavailable
|
|
|
|
WORK_TYPES = {
|
|
"bakalárska": "bachelor_thesis", "diplomová": "diploma_thesis", "dizertačná": "dissertation",
|
|
"habilitačná": "habilitation_thesis", "rigorózna": "doctoral_thesis", "záverečná stáž": "final_internship",
|
|
}
|
|
WORK_TYPE_LABELS = {
|
|
"bachelor_thesis": "bakalárska práca", "diploma_thesis": "diplomová práca", "dissertation": "dizertačná práca",
|
|
"habilitation_thesis": "habilitačná práca", "doctoral_thesis": "rigorózna práca",
|
|
"final_internship": "záverečná stáž", "other_monography": "iná práca",
|
|
}
|
|
_TOTAL = re.compile(r"Počet nájdených záznamov:\s*(\d+)")
|
|
_ROW_START = re.compile(r'<div id="\w+" class="well-sm horizontal-rule" >')
|
|
|
|
|
|
def text(value: str) -> str:
|
|
"""Visible text of an HTML fragment."""
|
|
value = re.sub(r"<br\s*/?>", " ", value)
|
|
value = re.sub(r"<[^>]+>", " ", value)
|
|
return re.sub(r"\s+", " ", html.unescape(value)).strip()
|
|
|
|
|
|
def fold(value: str) -> str:
|
|
"""Lowercase without diacritics (CRZP search is diacritics-insensitive too)."""
|
|
value = unicodedata.normalize("NFKD", value or "")
|
|
return "".join(c for c in value if not unicodedata.combining(c)).lower().strip()
|
|
|
|
|
|
def display_name(value: str) -> str:
|
|
""""Pták, Ján" -> "Ján Pták"; other forms unchanged."""
|
|
parts = [p.strip() for p in value.split(",") if p.strip()]
|
|
return " ".join(parts[1:] + parts[:1]) if len(parts) >= 2 else value.strip()
|
|
|
|
|
|
def work_type(label: str) -> str:
|
|
low = label.lower()
|
|
return next((v for k, v in WORK_TYPES.items() if k in low), "other_monography")
|
|
|
|
|
|
def split_school(value: str) -> tuple[str, str]:
|
|
"""School name and workplace part: "Technická univerzita v Košiciach 1040 104005" or
|
|
"... 1040 Katedra elektroniky ..." -> ("Technická univerzita v Košiciach", "1040 104005" / "1040 Katedra …")."""
|
|
value = " ".join(value.split())
|
|
match = re.search(r"\s(\d{3,}\b.*)$", value)
|
|
return (value[: match.start()].strip(), match.group(1).strip()) if match else (value, "")
|
|
|
|
|
|
def split_keywords(values: list[str]) -> list[str]:
|
|
"""Some schools send all tags as one "a; b; c" or "a, b, c" value, some even append
|
|
the abstract; such over-long pieces are not tags and are dropped."""
|
|
result = []
|
|
for value in values:
|
|
parts = re.split(r"[;|]", value)
|
|
if len(parts) == 1 and len(value) > 60 and "," in value:
|
|
parts = value.split(",")
|
|
result += [k.strip(" :;,.") for k in parts if k.strip(" :;,.") and len(k.strip()) <= 60]
|
|
return list(dict.fromkeys(result))
|
|
|
|
|
|
def facets(page: str) -> dict[str, list[dict]]:
|
|
"""Facet title -> [{label, count}] (CRZP shows the top values only)."""
|
|
result = {}
|
|
for raw in re.findall(r'<script type="application/json" id="props-\w+">(.*?)</script>', page, re.S):
|
|
try:
|
|
data = json.loads(raw.replace('\\\\"', '\\"'))
|
|
except ValueError:
|
|
continue
|
|
if isinstance(data, dict) and "items" in data and isinstance(data.get("strings"), dict):
|
|
result[data["strings"].get("title", "")] = [
|
|
{"label": i["label"], "count": int(i["count"])} for i in data["items"]]
|
|
return result
|
|
|
|
|
|
def facet_actions(page: str) -> dict[str, str]:
|
|
"""Facet title -> id of the element that shows all of its values."""
|
|
result = {}
|
|
for raw in re.findall(r'<script type="application/json" id="props-\w+">(.*?)</script>', page, re.S):
|
|
try:
|
|
data = json.loads(raw.replace('\\\\"', '\\"'))
|
|
except ValueError:
|
|
continue
|
|
show = data.get("onShowAll") if isinstance(data, dict) else None
|
|
if (isinstance(data.get("strings") if isinstance(data, dict) else None, dict) and isinstance(show, list)
|
|
and len(show) == 2 and isinstance(show[1], list) and show[1][1:2] == ["showMoreResultsNewAjax"]):
|
|
result[data["strings"].get("title", "")] = show[1][0]
|
|
return result
|
|
|
|
|
|
def _lines(chunk: str, label: str) -> list[str]:
|
|
return [text(v) for v in re.findall(re.escape(label) + r":\s*([^<]*)<", chunk) if text(v)]
|
|
|
|
|
|
def parse_results(page: str) -> dict:
|
|
total = _TOTAL.search(page)
|
|
if total is None:
|
|
raise CrzpUnavailable("page_structure_changed")
|
|
return {"total": int(total.group(1)), "facets": facets(page), "rows": parse_rows(page)}
|
|
|
|
|
|
def parse_rows(page: str) -> list[dict]:
|
|
"""Result rows of a full page or of an in-place list fragment."""
|
|
starts = [m.start() for m in _ROW_START.finditer(page)]
|
|
rows = []
|
|
for i, start in enumerate(starts):
|
|
chunk = page[start: starts[i + 1] if i + 1 < len(starts) else len(page)]
|
|
title = re.search(r'title="([^"]*)" onclick="javascript:ae\('\w+','navigate2DetailAjaxLong', chunk)
|
|
label = re.search(r'<span-label title="([^"]+)" class="label', chunk)
|
|
if not title or not label:
|
|
raise CrzpUnavailable("page_structure_changed")
|
|
school, unit = split_school((_lines(chunk, "Škola") or [""])[0])
|
|
year = re.search(r"Rok odovzdania:\s*(\d{4})", chunk)
|
|
rows.append({
|
|
"title": html.unescape(title.group(1)).strip(),
|
|
"status_label": html.unescape(label.group(1)),
|
|
"published": html.unescape(label.group(1)).lower().startswith("zverejnen"),
|
|
"work_type": work_type(label.group(1)),
|
|
"authors": _lines(chunk, "Autor"),
|
|
"supervisors": _lines(chunk, "Školiteľ"),
|
|
"opponents": _lines(chunk, "Oponent"),
|
|
"keywords": split_keywords([text(k) for k in re.findall(r'display-subject"[^>]*>(.*?)</a>', chunk)]),
|
|
"school": school,
|
|
"school_unit": unit,
|
|
"year": int(year.group(1)) if year else None,
|
|
})
|
|
return rows
|
|
|
|
|
|
def parse_rss(xml: str) -> list[dict]:
|
|
items = []
|
|
for title, link in re.findall(r"<item><title>(.*?)</title>.*?<link>(.*?)</link>", xml, re.S):
|
|
sid = re.search(r"sid=([0-9A-F]+)", html.unescape(link))
|
|
items.append({"title": html.unescape(title).strip(), "sid": sid.group(1) if sid else None})
|
|
return items
|
|
|
|
|
|
def _definitions(page: str) -> dict[str, str]:
|
|
result = {}
|
|
for term, value in re.findall(r"<dt[^>]*>(.*?)</dt>\s*<dd[^>]*>(.*?)</dd>", page, re.S):
|
|
if text(term):
|
|
result.setdefault(text(term), value)
|
|
return result
|
|
|
|
|
|
def parse_detail(page: str) -> dict:
|
|
sid = re.search(r"fn=detailBiblioForm&(?:amp;)?sid=([0-9A-F]+)", page)
|
|
title = re.search(r'<h4 id="\w+" class="inline bold" >(.*?)</h4>', page, re.S)
|
|
if not sid or not title:
|
|
raise CrzpUnavailable("page_structure_changed")
|
|
fields = _definitions(page)
|
|
|
|
def people(name):
|
|
return [text(p) for p in re.findall(r"<i>(.*?)</i>", fields.get(name, ""))]
|
|
|
|
school, unit = split_school(text(fields.get("Škola", "")))
|
|
pages = re.search(r"(\d+)", text(fields.get("Počet strán", "")))
|
|
year = re.search(r"(\d{4})", text(fields.get("Rok odovzdania", "")))
|
|
status = re.search(r'class="label label-\w+" >(.*?)</strong>', page)
|
|
keywords = []
|
|
for heading, body in re.findall(r"<b>(Kľúčové slová[^<]*)</b>.*?<div class=\"panel-body[^\"]*\">(.*?)</div></div>",
|
|
page, re.S):
|
|
keywords += [text(k) for k in re.findall(r"anKeywordClick'\);\" >(.*?)</a>", body) if text(k)]
|
|
|
|
def abstract(heading):
|
|
match = re.search(r"<strong>" + heading + r"</strong>.*?class=\"well well-sm\" >(.*?)</div>", page, re.S)
|
|
return text(match.group(1)) if match else ""
|
|
|
|
return {
|
|
"sid": sid.group(1),
|
|
"title": text(title.group(1)),
|
|
"status_label": re.sub(r"^\w+ - ", "", text(status.group(1))) if status else "",
|
|
"work_type": work_type(text(fields.get("Typ práce", "")) or (text(status.group(1)) if status else "")),
|
|
"authors": people("Autor"),
|
|
"supervisors": people("Školiteľ"),
|
|
"consultants": people("Konzultant"),
|
|
"opponents": people("Oponent"),
|
|
"school": school,
|
|
"school_unit": unit,
|
|
"year": int(year.group(1)) if year else None,
|
|
"pages": int(pages.group(1)) if pages else None,
|
|
"language": text(fields.get("Primárny jazyk", "")),
|
|
"field_of_study": text(fields.get("Študijný odbor", "")),
|
|
"keywords": split_keywords(keywords),
|
|
"abstract": abstract("Abstrakt v primárnom jazyku"),
|
|
"abstract_secondary": abstract("Abstrakt v sekundárnom jazyku"),
|
|
}
|