diff --git a/upraveny_pubcrawler/urls.py b/upraveny_pubcrawler/urls.py new file mode 100644 index 0000000..fddbf38 --- /dev/null +++ b/upraveny_pubcrawler/urls.py @@ -0,0 +1,54 @@ +import re +from urllib.parse import urlsplit +from ...common.urls import normalize_url + + + +ARTICLE_EXPERT = re.compile(r"^/ask-an-expert/[^/]+/[^/]+/?$", re.I) + +ARTICLE_FORUM = re.compile(r"^/forum/[^/]+/[^/]+/?$", re.I) + + +def normalize(url, base="https://www.modrykonik.sk/"): + return normalize_url(url, base=base, hosts={"www.modrykonik.sk", "modrykonik.sk"}, + canonical_host="www.modrykonik.sk") + + +def article_id(url): + path = urlsplit(url).path + m = ARTICLE_EXPERT.match(path) or ARTICLE_FORUM.match(path) + return m.group(0) if m else None + + +def kind(url): + parts = urlsplit(url) + path = parts.path.rstrip("/") or "/" + segments = [s.lower() for s in path.split("/") if s] + + if path == "/robots.txt": + return "robots" + if path.endswith((".xml", ".xml.gz")) and "sitemap" in path.lower(): + return "sitemap" + if not segments: + return None + + if segments[0] == "ask-an-expert": + if len(segments) == 1: + return "listing" + if len(segments) == 2: + return "listing" + if len(segments) == 3: + return "article_expert" + return None + + + if segments[0] == "forum": + if len(segments) == 1: + return "listing" + if len(segments) == 2: + return "listing" + if len(segments) == 3: + return "article_forum" + return None + + return None \ No newline at end of file