175 lines
4.1 KiB
Python
175 lines
4.1 KiB
Python
import hashlib
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
VSTUP = Path("./data/www.modrykonik.sk/forum")
|
|
VYSTUP = Path("./output")
|
|
|
|
|
|
CITLIVE_SLOVA = [
|
|
"samovraž", "zabiť sa", "zabit sa", "zomrieť", "zomriet",
|
|
"zdochla", "zdochnuť", "sebevraž", "suicid",
|
|
"kill myself", "end my life", "chcem umrieť", "chcem umriet",
|
|
]
|
|
|
|
|
|
MIN_DLZKA = 30
|
|
|
|
|
|
|
|
def vytiahni_data(subor):
|
|
|
|
try:
|
|
with open(subor, encoding="utf-8") as f:
|
|
html = f.read()
|
|
except Exception as e:
|
|
print(f"Nepodarilo sa otvoriť {subor}: {e}")
|
|
return None
|
|
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
|
|
for script in soup.find_all("script", type="application/ld+json"):
|
|
obsah = script.string
|
|
if not obsah:
|
|
continue
|
|
try:
|
|
data = json.loads(obsah)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(data, dict) and data.get("@type") == "DiscussionForumPosting":
|
|
return data
|
|
|
|
return None
|
|
|
|
|
|
|
|
def ocisti_text(s):
|
|
|
|
if not s:
|
|
return ""
|
|
s = re.sub(r"\s+", " ", s)
|
|
return s.strip()
|
|
|
|
|
|
|
|
def skry_meno(meno):
|
|
if not meno or meno == "anonymous_user":
|
|
return "anonymous_user"
|
|
h = hashlib.sha256(meno.encode("utf-8")).hexdigest()
|
|
return f"user_{h[:8]}"
|
|
|
|
|
|
def obsahuje_citlive(text):
|
|
text_lower = text.lower()
|
|
for slovo in CITLIVE_SLOVA:
|
|
if slovo in text_lower:
|
|
return True
|
|
return False
|
|
|
|
|
|
|
|
def vytvor_zaznam(data):
|
|
otazka = ocisti_text(data.get("text", ""))
|
|
if not otazka:
|
|
return None
|
|
if len(otazka) < MIN_DLZKA:
|
|
return None
|
|
if obsahuje_citlive(otazka):
|
|
return None
|
|
|
|
|
|
komentare = data.get("comment", []) or []
|
|
if not komentare:
|
|
return None
|
|
|
|
casti_odpovede = []
|
|
autori = set()
|
|
|
|
for kom in komentare:
|
|
text = ocisti_text(kom.get("text", ""))
|
|
if not text or len(text) < MIN_DLZKA:
|
|
continue
|
|
if obsahuje_citlive(text):
|
|
continue
|
|
|
|
casti_odpovede.append(text)
|
|
|
|
|
|
autor = kom.get("author", {}).get("name", "")
|
|
autori.add(skry_meno(autor))
|
|
|
|
if not casti_odpovede:
|
|
return None
|
|
|
|
|
|
odpoved = "\n\n".join(casti_odpovede[:5])
|
|
|
|
return {
|
|
"messages": [
|
|
{"role": "user", "content": otazka},
|
|
{"role": "assistant", "content": odpoved},
|
|
],
|
|
"headline": ocisti_text(data.get("headline", "")),
|
|
"url": data.get("url", ""),
|
|
"num_comments": len(komentare),
|
|
"authors": sorted(autori),
|
|
}
|
|
|
|
|
|
def main():
|
|
VYSTUP.mkdir(exist_ok=True)
|
|
|
|
subory = []
|
|
for subor in VSTUP.rglob("index.html"):
|
|
subory.append(subor)
|
|
|
|
print(f"Našiel som {len(subory)} HTML súborov")
|
|
|
|
zaznamy = []
|
|
pocet_citlivych = 0
|
|
pocet_kratkych = 0
|
|
|
|
for i, subor in enumerate(subory, 1):
|
|
if i % 10 == 0:
|
|
print(f" Spracovávam {i}/{len(subory)}...")
|
|
|
|
data = vytiahni_data(subor)
|
|
if not data:
|
|
continue
|
|
|
|
zaznam = vytvor_zaznam(data)
|
|
if zaznam:
|
|
zaznamy.append(zaznam)
|
|
else:
|
|
otazka = ocisti_text(data.get("text", ""))
|
|
if otazka and obsahuje_citlive(otazka):
|
|
pocet_citlivych += 1
|
|
else:
|
|
pocet_kratkych += 1
|
|
|
|
print(f"\nVýsledok:")
|
|
print(f" Platných vlákien: {len(zaznamy)}")
|
|
print(f" Preskočených (citlivé): {pocet_citlivych}")
|
|
print(f" Preskočených (krátke/prázdne): {pocet_kratkych}")
|
|
|
|
if not zaznamy:
|
|
print("Nenašiel som žiadne dáta, končím.")
|
|
return
|
|
|
|
vystupny_subor = VYSTUP / "modry_konik.jsonl"
|
|
with open(vystupny_subor, "w", encoding="utf-8") as f:
|
|
for z in zaznamy:
|
|
f.write(json.dumps(z, ensure_ascii=False) + "\n")
|
|
|
|
print(f"\nUložené do: {vystupny_subor}")
|
|
|
|
print("\nUkážka prvého záznamu:")
|
|
print(json.dumps(zaznamy[0], ensure_ascii=False, indent=2)[:1500])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |