merge
This commit is contained in:
commit
75840f6d21
@ -4,6 +4,7 @@ import trafilatura.feeds
|
||||
import trafilatura.sitemaps
|
||||
import trafilatura.spider
|
||||
import trafilatura.utils
|
||||
import trafilatura.external
|
||||
import sys
|
||||
import courlan
|
||||
import urllib
|
||||
@ -11,6 +12,7 @@ from datetime import datetime
|
||||
import click
|
||||
import logging as LOGGER
|
||||
import os
|
||||
import pprint
|
||||
|
||||
LANGUAGE= os.getenv("SUCKER_LANGUAGE","sk")
|
||||
DOMAIN = os.getenv("SUCKER_DOMAIN","sk")
|
||||
@ -21,6 +23,18 @@ MINFILESIZE=300
|
||||
MAXFILESIZE=10000000
|
||||
MINTEXTSIZE=200
|
||||
|
||||
def put_queue(db,channel,message):
|
||||
queuecol = db["queue"]
|
||||
queuecol.insert_one({"channel":channel,"message":message,"created_at":datetime.utcnow(),"started_at":None})
|
||||
|
||||
def reserve_queue(db,channel,message):
|
||||
queuecol = db["queue"]
|
||||
r = queuecol.find_one_and_delete({"channel":channel},sort={"created_at":-1})
|
||||
|
||||
def delete_queue(db,channel):
|
||||
queuecol = db["queue"]
|
||||
pass
|
||||
|
||||
def calculate_checksums(text):
|
||||
"""
|
||||
@return fingerprints of a paragraphs in text. Paragraphs are separated by a blank line
|
||||
@ -61,30 +75,17 @@ def is_link_good(link):
|
||||
if r is None:
|
||||
#print(link)
|
||||
return None
|
||||
llink,ldomain = r
|
||||
#print(llink,ldomain)
|
||||
# domain rules
|
||||
if not ldomain.endswith(DOMAIN):
|
||||
LOGGER.debug("bad domain")
|
||||
llink,lhostname = r
|
||||
#print(llink,lhostname)
|
||||
# hostname rules
|
||||
if not lhostname.endswith(DOMAIN):
|
||||
LOGGER.debug("bad hostname")
|
||||
return None
|
||||
if courlan.is_not_crawlable(llink):
|
||||
LOGGER.debug("not crawlable")
|
||||
return None
|
||||
return llink
|
||||
|
||||
def filter_links(links,rules=None):
|
||||
out = set()
|
||||
for link in links:
|
||||
r = is_link_good(link)
|
||||
if r is None:
|
||||
continue
|
||||
# check robots.txt rules
|
||||
if rules is not None and not rules.can_fetch("*", r):
|
||||
continue
|
||||
out.add(llink)
|
||||
return out
|
||||
|
||||
|
||||
def get_link_doc(link,status="frontlink"):
|
||||
r = courlan.check_url(link)
|
||||
assert r is not None
|
||||
@ -148,15 +149,18 @@ def extract_pages(link_batch,responses):
|
||||
assert original_link is not None
|
||||
if html is not None:
|
||||
doc = trafilatura.bare_extraction(html,url=final_link,with_metadata=True,include_formatting=True,target_language=LANGUAGE)
|
||||
print("html2doc")
|
||||
print(text)
|
||||
if doc is not None:
|
||||
if not "text" in doc or len(doc["text"]) < MINTEXTSIZE:
|
||||
# text too small
|
||||
doc = None
|
||||
|
||||
out.append((original_link,final_link,html,doc))
|
||||
return out
|
||||
|
||||
|
||||
def index_pages(db,domain,extracted_pages):
|
||||
def index_pages(db,hostname,extracted_pages):
|
||||
linkcol = db["links"]
|
||||
htmlcol = db["html"]
|
||||
contentcol = db["content"]
|
||||
@ -179,7 +183,7 @@ def index_pages(db,domain,extracted_pages):
|
||||
doc["paragraph_sizes"] = sizes
|
||||
goodsz = sum(sizes)
|
||||
if len(text) < 200 or goodsz/len(text) < 0.4:
|
||||
state = "trash"
|
||||
stat = "trash"
|
||||
if state == "good":
|
||||
htdoc = get_link_doc(link,state)
|
||||
htdoc["html"] = html
|
||||
@ -195,7 +199,7 @@ def index_pages(db,domain,extracted_pages):
|
||||
linkcol.update_one({"url":original_link},{"$set":{"status":state}})
|
||||
|
||||
|
||||
def extract_links(link_batch,responses,domain,rules,default_status="frontlink"):
|
||||
def extract_links(link_batch,responses,hostname,rules,default_status="frontlink"):
|
||||
links = {}
|
||||
for original_link,(final_link,html) in zip(link_batch,responses):
|
||||
status = default_status
|
||||
@ -234,9 +238,9 @@ def index_links(db,extracted_links):
|
||||
except pymongo.errors.DuplicateKeyError as ex:
|
||||
pass
|
||||
|
||||
def get_links(db,domain,status,batch_size=BATCHSIZE):
|
||||
def get_links(db,hostname,status,batch_size=BATCHSIZE):
|
||||
linkcol = db["links"]
|
||||
res = linkcol.find({"status":status,"host":domain},{"url":1},limit=batch_size)
|
||||
res = linkcol.find({"status":status,"host":hostname},{"url":1},limit=batch_size)
|
||||
links = []
|
||||
for doc in res:
|
||||
#print(">>>>>" + status)
|
||||
@ -246,32 +250,32 @@ def get_links(db,domain,status,batch_size=BATCHSIZE):
|
||||
|
||||
|
||||
|
||||
def process_links(db,domain,status,links=[],rules=None,batch_size=BATCHSIZE):
|
||||
def process_links(db,hostname,status,links=[],rules=None,batch_size=BATCHSIZE):
|
||||
#print(links)
|
||||
responses = fetch_pages(links)
|
||||
#print(responses)
|
||||
extracted_pages = extract_pages(links,responses)
|
||||
#print(extracted_pages)
|
||||
extracted_links = extract_links(links,responses,domain,rules,status)
|
||||
extracted_links = extract_links(links,responses,hostname,rules,status)
|
||||
#print(extracted_links)
|
||||
index_links(db,extracted_links)
|
||||
index_pages(db,domain,extracted_pages)
|
||||
index_pages(db,hostname,extracted_pages)
|
||||
|
||||
|
||||
def link_summary(db,domain):
|
||||
def link_summary(db,hostname):
|
||||
linkcol = db["links"]
|
||||
#res = linkcol.distinct("domain",{"hostname":domain})
|
||||
#res = linkcol.distinct("hostname",{"hostname":hostname})
|
||||
|
||||
# count links
|
||||
res = linkcol.aggregate([
|
||||
{"$match":{"host":domain}},
|
||||
{"$match":{"host":hostname}},
|
||||
{"$group":{"_id":"$status","count":{"$sum":1}}},
|
||||
])
|
||||
for item in res:
|
||||
print(item)
|
||||
contentcol = db["content"]
|
||||
res = contentcol.aggregate([
|
||||
{"$match":{"hostname":domain}},
|
||||
{"$match":{"hostname":hostname}},
|
||||
{"$group":{"_id":None,"text_size_sum":{"$sum":"text_size"}}},
|
||||
])
|
||||
for item in res:
|
||||
@ -292,7 +296,7 @@ def createdb():
|
||||
contentcol = db["content"]
|
||||
contentcol.create_index("url",unique=True)
|
||||
#contentcol.create_index({"paragraph_checksums":1})
|
||||
#contentcol.create_index({"domain":1})
|
||||
#contentcol.create_index({"hostname":1})
|
||||
htmlcol = db["html"]
|
||||
htmlcol.create_index("url",unique=True)
|
||||
|
||||
@ -319,30 +323,44 @@ def externaldomains(link):
|
||||
for d in domains:
|
||||
print(d)
|
||||
|
||||
@cli.command()
|
||||
@click.argument("start_link")
|
||||
def parseurl(start_link):
|
||||
link,hostname = courlan.check_url(start_link)
|
||||
links = [link]
|
||||
responses = fetch_pages(links)
|
||||
#pprint.pprint(responses)
|
||||
extracted_pages = extract_pages(links,responses)
|
||||
for ol,bl,html,doc in extracted_pages:
|
||||
pprint.pprint(doc)
|
||||
extracted_links = extract_links(links,responses,hostname,None,"backlink")
|
||||
pprint.pprint(extracted_links)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument("start_link")
|
||||
def visit(start_link):
|
||||
myclient = pymongo.MongoClient(CONNECTION)
|
||||
db=myclient[DBNAME]
|
||||
start_link,domain = courlan.check_url(start_link)
|
||||
rules = fetch_robot(domain)
|
||||
start_link,hostname = courlan.check_url(start_link)
|
||||
rules = fetch_robot(hostname)
|
||||
print(rules)
|
||||
batch_size = BATCHSIZE
|
||||
navigation_links = get_links(db,domain,"navigation",batch_size)
|
||||
navigation_links = get_links(db,hostname,"navigation",batch_size)
|
||||
if start_link is not None:
|
||||
navigation_links.append(start_link)
|
||||
print(f"Navigation links {len(navigation_links)}")
|
||||
process_links(db,domain,"frontlink",navigation_links,rules)
|
||||
links = get_links(db,domain,"frontlink",batch_size)
|
||||
process_links(db,hostname,"frontlink",navigation_links,rules)
|
||||
links = get_links(db,hostname,"frontlink",batch_size)
|
||||
bl = len(links) - batch_size
|
||||
print(f"Got {len(links)} frontlinks")
|
||||
if bl > 0:
|
||||
print("Getting backlinks")
|
||||
front_links = get_links(db,domain,"backlink",bl)
|
||||
front_links = get_links(db,hostname,"backlink",bl)
|
||||
links += front_links
|
||||
print("Processing backlinks")
|
||||
process_links(db,domain,"backlink",links,rules=rules)
|
||||
link_summary(db,domain)
|
||||
process_links(db,hostname,"backlink",links,rules=rules)
|
||||
link_summary(db,hostname)
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user