dano/websucker-pip
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zz

Browse Source
This commit is contained in:
Daniel Hládek 2023-03-12 06:16:47 +01:00
parent 39f66bc98a
commit e728134bf6
1 changed files with 37 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
mongo/mongocwarler.py
View File

@ -7,12 +7,15 @@ import trafilatura.utils
import sys
import courlan
import urllib
from datetime import datetime
import click
LANGUAGE="sk"
DOMAIN = "sk"
BATCHSIZE=10
MINFILESIZE=300
MAXFILESIZE=1000000
MAXFILESIZE=10000000
MINTEXTSIZE=200
def calculate_checksums(text):
"""
@ -83,7 +86,7 @@ def get_link_doc(link,status="frontlink"):
assert r is not None
link,host = r
domain = courlan.extract_domain(link)
return {"url":link,"host":host,"domain":domain,"status":status}
return {"url":link,"host":host,"domain":domain,"status":status,"created_at":datetime.utcnow()}
def generic_visit(domain):
known_links = set(get_visited_links(domain))
@ -147,6 +150,10 @@ def extract_pages(link_batch,responses):
assert original_link is not None
if html is not None:
doc = trafilatura.bare_extraction(html,url=final_link,with_metadata=True,include_formatting=True,target_language=LANGUAGE)
if doc is not None:
if not "text" in doc or len(doc["text"]) < MINTEXTSIZE:
# text too small
doc = None
out.append((original_link,final_link,html,doc))
return out
@ -167,10 +174,12 @@ def index_pages(db,domain,extracted_pages):
linkcol.insert_one(get_link_doc(final_link,state))
state = "redirect"
linkcol.update_one({"url":original_link},{"$set":{"status":state}})
if html is not None:
htmlcol.insert_one({"url":final_link,"html":html})
if doc is not None:
if html is not None:
htmlcol.insert_one({"url":final_link,"html":html,"html_size":len(html),"created_at":datetime.utcnow()})
checksums,sizes = calculate_checksums(doc["text"])
doc["created_at"] = datetime.utcnow()
doc["text_size"] = len(doc["text"])
doc["paragraph_checksums"] = checksums
doc["paragraph_sizes"] = sizes
# todo extract links
@ -231,6 +240,30 @@ def process_links(db,domain,status,links=[],rules=None,batch_size=BATCHSIZE):
index_links(db,extracted_links)
index_pages(db,domain,extracted_pages)
def link_summary(db,domain):
linkcol = db["links"]
#res = linkcol.distinct("domain",{"hostname":domain})
res = linkcol.aggregate([
{"$match":{"host":domain}},
{"$group":{"_id":"$status","count":{"$sum":1}}},
])
for item in res:
print(item)
def create_indices(db):
linkcol = db["links"]
linkcol.create_index({"url":1},{"name":"url"})
linkcol.create_index({"host":1,"status":1},{"name":"hostname_status"})
contentcol = db["content"]
contentcol.create_index({"url":1})
contentcol.create_index({"paragraph_checksums":1})
contentcol.create_index({"domain":1})
htmlcol = db["html"]
htmlcol.create_index({"url":1})
@click.command()
def simple_visit(start_link=None):
start_link,domain = courlan.check_url(start_link)
myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/")
@ -249,25 +282,3 @@ def simple_visit(start_link=None):
front_links = get_links(db,domain,"backlink",bl)
process_links(db,domain,"backlink",links,rules=rules)
link_summary(db,domain)
def link_summary(db,domain):
linkcol = db["links"]
res = linkcol.aggregate([
{"$match":{"hostname":domain}},
{"$group":{"_id":{"status":domain},"count":{"$count":1}}},
])
print(res)
def create_indices(db):
linkcol = db["links"]
linkcol.create_index({"url":1},{"name":"url"})
linkcol.create_index({"hostname":1,"status":1},{"name":"hostname_status"})
contentcol = db["content"]
contentcol.create_index({"url":1})
contentcol.create_index({"paragraph_checksums":1})
contentcol.create_index({"domain":1})
htmlcol = db["html"]
htmlcol.create_index({"url":1})
simple_visit(sys.argv[1])