dano/websucker-pip
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zz

Browse Source
This commit is contained in:
Daniel Hládek 2023-03-17 12:30:53 +01:00
parent 0accadf633
commit 399f263921
1 changed files with 31 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
mongo/mongocwarler.py
View File

@ -63,30 +63,17 @@ def is_link_good(link):
if r is None:
#print(link)
return None
llink,ldomain = r
#print(llink,ldomain)
# domain rules
if not ldomain.endswith(DOMAIN):
LOGGER.debug("bad domain")
llink,lhostname = r
#print(llink,lhostname)
# hostname rules
if not lhostname.endswith(DOMAIN):
LOGGER.debug("bad hostname")
return None
if courlan.is_not_crawlable(llink):
LOGGER.debug("not crawlable")
return None
return llink
def filter_links(links,rules=None):
out = set()
for link in links:
r = is_link_good(link)
if r is None:
continue
# check robots.txt rules
if rules is not None and not rules.can_fetch("*", r):
continue
out.add(llink)
return out
def get_link_doc(link,status="frontlink"):
r = courlan.check_url(link)
assert r is not None
@ -94,12 +81,12 @@ def get_link_doc(link,status="frontlink"):
domain = courlan.extract_domain(link)
return {"url":link,"host":host,"domain":domain,"status":status,"created_at":datetime.utcnow()}
def generic_visit(domain):
known_links = set(get_visited_links(domain))
def generic_visit(hostname):
known_links = set(get_visited_links(hostname))
visit_links = []
visit_links = trafilatura.find_feed_urls(domain)
visit_links = trafilatura.find_feed_urls(hostname)
if visit_links is None:
visit_links = trafilatura.sitemap_search(domain)
visit_links = trafilatura.sitemap_search(hostname)
if visit_links is None:
visit_links = trafilatura.focused_crawler(dommain,known_links=known_links)
@ -170,7 +157,7 @@ def extract_pages(link_batch,responses):
return out
def index_pages(db,domain,extracted_pages):
def index_pages(db,hostname,extracted_pages):
linkcol = db["links"]
htmlcol = db["html"]
contentcol = db["content"]
@ -209,7 +196,7 @@ def index_pages(db,domain,extracted_pages):
linkcol.update_one({"url":original_link},{"$set":{"status":state}})
def extract_links(link_batch,responses,domain,rules,default_status="frontlink"):
def extract_links(link_batch,responses,hostname,rules,default_status="frontlink"):
links = {}
for original_link,(final_link,html) in zip(link_batch,responses):
status = default_status
@ -249,9 +236,9 @@ def index_links(db,extracted_links):
pass
def get_links(db,domain,status,batch_size=BATCHSIZE):
def get_links(db,hostname,status,batch_size=BATCHSIZE):
linkcol = db["links"]
res = linkcol.find({"status":status,"host":domain},{"url":1},limit=batch_size)
res = linkcol.find({"status":status,"host":hostname},{"url":1},limit=batch_size)
links = []
for doc in res:
#print(">>>>>" + status)
@ -261,32 +248,32 @@ def get_links(db,domain,status,batch_size=BATCHSIZE):
def process_links(db,domain,status,links=[],rules=None,batch_size=BATCHSIZE):
def process_links(db,hostname,status,links=[],rules=None,batch_size=BATCHSIZE):
#print(links)
responses = fetch_pages(links)
#print(responses)
extracted_pages = extract_pages(links,responses)
#print(extracted_pages)
extracted_links = extract_links(links,responses,domain,rules,status)
extracted_links = extract_links(links,responses,hostname,rules,status)
#print(extracted_links)
index_links(db,extracted_links)
index_pages(db,domain,extracted_pages)
index_pages(db,hostname,extracted_pages)
def link_summary(db,domain):
def link_summary(db,hostname):
linkcol = db["links"]
#res = linkcol.distinct("domain",{"hostname":domain})
#res = linkcol.distinct("hostname",{"hostname":hostname})
# count links
res = linkcol.aggregate([
{"$match":{"host":domain}},
{"$match":{"host":hostname}},
{"$group":{"_id":"$status","count":{"$sum":1}}},
])
for item in res:
print(item)
contentcol = db["content"]
res = contentcol.aggregate([
{"$match":{"hostname":domain}},
{"$match":{"hostname":hostname}},
{"$group":{"_id":None,"text_size_sum":{"$sum":"text_size"}}},
])
for item in res:
@ -307,21 +294,21 @@ def createdb():
contentcol = db["content"]
contentcol.create_index("url",unique=True)
#contentcol.create_index({"paragraph_checksums":1})
#contentcol.create_index({"domain":1})
#contentcol.create_index({"hostname":1})
htmlcol = db["html"]
htmlcol.create_index("url",unique=True)
@cli.command()
@click.argument("start_link")
def parseurl(start_link):
link,domain = courlan.check_url(start_link)
link,hostname = courlan.check_url(start_link)
links = [link]
responses = fetch_pages(links)
#pprint.pprint(responses)
extracted_pages = extract_pages(links,responses)
for ol,bl,html,doc in extracted_pages:
pprint.pprint(doc)
extracted_links = extract_links(links,responses,domain,None,"backlink")
extracted_links = extract_links(links,responses,hostname,None,"backlink")
pprint.pprint(extracted_links)
@ -330,25 +317,25 @@ def parseurl(start_link):
def visit(start_link):
myclient = pymongo.MongoClient(CONNECTION)
db=myclient[DBNAME]
start_link,domain = courlan.check_url(start_link)
rules = fetch_robot(domain)
start_link,hostname = courlan.check_url(start_link)
rules = fetch_robot(hostname)
print(rules)
batch_size = BATCHSIZE
navigation_links = get_links(db,domain,"navigation",batch_size)
navigation_links = get_links(db,hostname,"navigation",batch_size)
if start_link is not None:
navigation_links.append(start_link)
print(f"Navigation links {len(navigation_links)}")
process_links(db,domain,"frontlink",navigation_links,rules)
links = get_links(db,domain,"frontlink",batch_size)
process_links(db,hostname,"frontlink",navigation_links,rules)
links = get_links(db,hostname,"frontlink",batch_size)
bl = len(links) - batch_size
print(f"Got {len(links)} frontlinks")
if bl > 0:
print("Getting backlinks")
front_links = get_links(db,domain,"backlink",bl)
front_links = get_links(db,hostname,"backlink",bl)
links += front_links
print("Processing backlinks")
process_links(db,domain,"backlink",links,rules=rules)
link_summary(db,domain)
process_links(db,hostname,"backlink",links,rules=rules)
link_summary(db,hostname)
if __name__ == "__main__":
cli()