From c657e52410c0d9c8c73885b10fd8aed0fec48498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristi=C3=A1n=20Sokolsk=C3=BD?= Date: Fri, 6 Nov 2026 11:10:23 +0000 Subject: [PATCH] =?UTF-8?q?P=C5=99idat=20upraveny=5Fpubcrawler/urls.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upraveny_pubcrawler/urls.py | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 upraveny_pubcrawler/urls.py diff --git a/upraveny_pubcrawler/urls.py b/upraveny_pubcrawler/urls.py new file mode 100644 index 0000000..fddbf38 --- /dev/null +++ b/upraveny_pubcrawler/urls.py @@ -0,0 +1,54 @@ +import re +from urllib.parse import urlsplit +from ...common.urls import normalize_url + + + +ARTICLE_EXPERT = re.compile(r"^/ask-an-expert/[^/]+/[^/]+/?$", re.I) + +ARTICLE_FORUM = re.compile(r"^/forum/[^/]+/[^/]+/?$", re.I) + + +def normalize(url, base="https://www.modrykonik.sk/"): + return normalize_url(url, base=base, hosts={"www.modrykonik.sk", "modrykonik.sk"}, + canonical_host="www.modrykonik.sk") + + +def article_id(url): + path = urlsplit(url).path + m = ARTICLE_EXPERT.match(path) or ARTICLE_FORUM.match(path) + return m.group(0) if m else None + + +def kind(url): + parts = urlsplit(url) + path = parts.path.rstrip("/") or "/" + segments = [s.lower() for s in path.split("/") if s] + + if path == "/robots.txt": + return "robots" + if path.endswith((".xml", ".xml.gz")) and "sitemap" in path.lower(): + return "sitemap" + if not segments: + return None + + if segments[0] == "ask-an-expert": + if len(segments) == 1: + return "listing" + if len(segments) == 2: + return "listing" + if len(segments) == 3: + return "article_expert" + return None + + + if segments[0] == "forum": + if len(segments) == 1: + return "listing" + if len(segments) == 2: + return "listing" + if len(segments) == 3: + return "article_forum" + return None + + return None \ No newline at end of file