dano/websucker-pip
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merge

Browse Source
This commit is contained in:
Daniel Hládek 2023-04-06 17:06:08 +02:00
commit e06ef64c8f
4 changed files with 218 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
mongo/Dockerfile Normal file
View File

@ -0,0 +1,7 @@
FROM python:3.9
RUN mkdir /app
COPY requirements.txt /app
RUN pip install -r /app/requirements.txt
COPY *.py /app
WORKDIR /app
ENTRYPOINT ["python", "./mongocrawler.py"]

363
mongo/mongocwarler.py → mongo/mongocrawler.py
View File

@ -18,11 +18,12 @@ import re
import time import time
import collections import collections
import math import math
import json import random
import hashlib
LANGUAGE= os.getenv("SUCKER_LANGUAGE","sk") LANGUAGE= os.getenv("SUCKER_LANGUAGE","sk")
DOMAIN = os.getenv("SUCKER_DOMAIN","sk") DOMAIN = os.getenv("SUCKER_DOMAIN","sk")
BATCHSIZE=os.getenv("SUCKER_BATCHSIZE",10) BATCHSIZE=int(os.getenv("SUCKER_BATCHSIZE","10"))
CONNECTION=os.getenv("SUCKER_CONNECTION","mongodb://root:example@localhost:27017/") CONNECTION=os.getenv("SUCKER_CONNECTION","mongodb://root:example@localhost:27017/")
DBNAME=os.getenv("SUCKER_DBNAME","crawler") DBNAME=os.getenv("SUCKER_DBNAME","crawler")
MINFILESIZE=300 MINFILESIZE=300
@ -32,17 +33,15 @@ CHECK_PARAGRAPH_SIZE=150
TEXT_TRASH_SIZE=200 TEXT_TRASH_SIZE=200
TEXT_TRASH_RATIO=0.6 TEXT_TRASH_RATIO=0.6
def put_queue(db,channel,message): def split_train(res):
queuecol = db["queue"] trainset = []
queuecol.insert_one({"channel":channel,"message":message,"created_at":datetime.utcnow(),"started_at":None}) testset = []
for i,item in enumerate(res):
def reserve_queue(db,channel,message): if i % 10 == 0:
queuecol = db["queue"] testset.append(item)
r = queuecol.find_one_and_delete({"channel":channel},sort={"created_at":-1}) else:
trainset.append(item)
def delete_queue(db,channel): return trainset,testset
queuecol = db["queue"]
pass
def calculate_checksums(text): def calculate_checksums(text):
""" """
@ -102,39 +101,34 @@ def get_link_doc(link,status="frontlink"):
return {"url":link,"host":host,"domain":domain,"status":status,"created_at":datetime.utcnow()} return {"url":link,"host":host,"domain":domain,"status":status,"created_at":datetime.utcnow()}
def fetch_pages(link_batch): def fetch_page(link):
htmls = [] print("fetching:::::")
#print(link_batch) print(link)
#print("zzzzzzzzzz") final_link = link
for link in link_batch: response = trafilatura.fetch_url(link,decode=False)
print("fetching:::::") time.sleep(2)
print(link) html = None
final_link = link if response is not None :
response = trafilatura.fetch_url(link,decode=False) good = True
time.sleep(2) if response.status != 200:
html = None good = False
if response is not None : LOGGER.error('not a 200 response: %s for URL %s', response.status, url)
good = True elif response.data is None or len(response.data) < MINFILESIZE:
if response.status != 200: LOGGER.error('too small/incorrect for URL %s', link)
good = False good = False
LOGGER.error('not a 200 response: %s for URL %s', response.status, url) # raise error instead?
elif response.data is None or len(response.data) < MINFILESIZE: elif len(response.data) > MAXFILESIZE:
LOGGER.error('too small/incorrect for URL %s', link) good = False
good = False LOGGER.error('too large: length %s for URL %s', len(response.data), link)
# raise error instead? if good:
elif len(response.data) > MAXFILESIZE: html = trafilatura.utils.decode_response(response)
good = False final_link = response.url
LOGGER.error('too large: length %s for URL %s', len(response.data), link) if html is not None:
if good: html, final_link = trafilatura.spider.refresh_detection(html, final_link)
html = trafilatura.utils.decode_response(response) # is there a meta-refresh on the page?
final_link = response.url if final_link is None: # malformed or malicious content
if html is not None: html = None
html, final_link = trafilatura.spider.refresh_detection(html, final_link) return final_link,html
# is there a meta-refresh on the page?
if final_link is None: # malformed or malicious content
html = None
htmls.append((final_link,html))
return htmls
def fetch_robot(base_url): def fetch_robot(base_url):
try: try:
@ -186,6 +180,7 @@ def index_pages(db,hostname,extracted_pages):
text = doc["text"] text = doc["text"]
checksums,sizes = calculate_checksums(text) checksums,sizes = calculate_checksums(text)
doc["text_size"] = len(text) doc["text_size"] = len(text)
doc["text_md5"] = hashlib.md5(text.encode("utf8")).hexdigest()
doc["paragraph_checksums"] = checksums doc["paragraph_checksums"] = checksums
doc["paragraph_sizes"] = sizes doc["paragraph_sizes"] = sizes
goodsz = sum(sizes) goodsz = sum(sizes)
@ -214,6 +209,7 @@ def index_pages(db,hostname,extracted_pages):
htdoc = get_link_doc(link,state) htdoc = get_link_doc(link,state)
htdoc["html"] = html htdoc["html"] = html
htdoc["html_size"] = len(html) htdoc["html_size"] = len(html)
htdoc["html_md5"]= hashlib.md5(html.encode("utf8")).hexdigest()
# can be revisited - upsert # can be revisited - upsert
del htdoc["url"] del htdoc["url"]
htmlcol.update_one({"url":link},{"$set":htdoc},upsert=True) htmlcol.update_one({"url":link},{"$set":htdoc},upsert=True)
@ -227,7 +223,7 @@ def index_pages(db,hostname,extracted_pages):
checkcol.insert_one({"_id":chs}) checkcol.insert_one({"_id":chs})
except pymongo.errors.DuplicateKeyError as err: except pymongo.errors.DuplicateKeyError as err:
pass pass
linkcol.update_one({"url":original_link},{"$set":{"status":state}}) linkcol.update_one({"url":link},{"$set":{"status":state}})
def extract_links(link_batch,responses,hostname,rules,default_status="frontlink"): def extract_links(link_batch,responses,hostname,rules,default_status="frontlink"):
@ -263,23 +259,27 @@ def index_links(db,extracted_links):
for link,status in extracted_links: for link,status in extracted_links:
if not is_link_good(link): if not is_link_good(link):
continue continue
doc = get_link_doc(link,status) if status == "frontlink" or status == "backlink":
try: doc = get_link_doc(link,status)
linkcol.insert_one(doc) try:
except pymongo.errors.DuplicateKeyError as ex: linkcol.insert_one(doc)
pass # dont overwrite
except pymongo.errors.DuplicateKeyError as ex:
pass
else:
print("updating " + link,status)
linkcol.update_one({"url":link},{"$set":{"status":status,"updated_at":datetime.utcnow()}})
def get_link_features(link): def get_link_features(link):
a, urlpath = courlan.get_host_and_path(link) a, urlpath = courlan.get_host_and_path(link)
features = re.split("[/?&]",urlpath) features = re.split("[/?&]",urlpath)
#features = re.split("[/?-_=]",urlpath) #features = re.split("[/?-_=]",urlpath)
res = [] res = []
for feature in features: for i,feature in enumerate(features):
if len(feature) < 1: if len(feature) < 1:
continue continue
if feature.isdigit(): feature = re.sub("[0-9]","*",feature)
feature = "<NUM>" res.append(str(i)+ "-" + feature)
res.append(feature)
if len(res) < 2: if len(res) < 2:
return None return None
res = res[:-1] res = res[:-1]
@ -295,20 +295,14 @@ class LinkClassifier:
self.bad_count = 0 self.bad_count = 0
self.alpha = 0.001 self.alpha = 0.001
def train(self,db,hostname): def train(self,links):
linkcol = db["links"] for i,item in enumerate(links):
res = linkcol.find({"host":hostname,"status": {"$not":{"$in":["frontlink","backlink"]}}})
testset = []
for i,item in enumerate(res):
link = item["url"] link = item["url"]
state = item["status"] state = item["status"]
cl = 0 cl = 0
if state == "good": if state == "good":
cl = 1 cl = 1
print(cl,state,link) print(cl,state,link)
if i % 10 == 1:
testset.append((link,cl))
continue
features = get_link_features(link) features = get_link_features(link)
if features is None: if features is None:
continue continue
@ -323,22 +317,42 @@ class LinkClassifier:
self.badcounter[feature] += 1 self.badcounter[feature] += 1
self.bdictsize = len(self.badcounter) self.bdictsize = len(self.badcounter)
self.gdictsize = len(self.goodcounter) self.gdictsize = len(self.goodcounter)
def test(self,testset):
# eval # eval
gg = 0 gg = 0
for l,cl in testset: true_positive = 0
positive = 0
false_negative = 0
for item in testset:
l = item["url"]
cl = 0
if item["status"] == "good":
cl = 1
pcp = self.classify(l) pcp = self.classify(l)
r = 0 r = 0
if pcp > 0: if pcp > 0:
r = 1 r = 1
if cl == 1:
if r == 1:
true_positive += 1
positive += 1
if r == 1 and cl == 0:
false_negative += 1
if r == cl: if r == cl:
gg += 1 gg += 1
else: else:
print("MISS",l,cl,pcp) print("MISS",l,cl,pcp)
print("Accuracy:")
print(len(testset)) print(len(testset))
print(gg / len(testset)) print("Precision: {}, Recall: {}".format(true_positive/positive,true_positive/(true_positive+false_negative)))
print("Accuracy:")
acc = gg / len(testset)
print(acc)
return acc
def classify(self,link): def classify(self,link):
if self.good_count == 0 or self.bad_count == 0:
return random.uniform(-0.1,0.1)
features = get_link_features(link) features = get_link_features(link)
res = 0 res = 0
gp = math.log(self.good_count) - math.log(self.good_count + self.bad_count) gp = math.log(self.good_count) - math.log(self.good_count + self.bad_count)
@ -350,64 +364,25 @@ class LinkClassifier:
goodprob = 0 goodprob = 0
badprob = 0 badprob = 0
for feature in features: for feature in features:
g = math.log((self.goodcounter[feature] + self.alpha)) - gcc g = math.log((self.goodcounter[feature] + self.alpha)) - gcc
goodprob += g goodprob += g
b = math.log(self.badcounter[feature] + self.alpha) - bcc b = math.log(self.badcounter[feature] + self.alpha) - bcc
badprob += b badprob += b
print(feature,g,b) print(feature,g,b)
if (goodprob + gp) > (badprob + bp):
#if goodprob > badprob:
res = 1
pa = math.exp(goodprob + gp) pa = math.exp(goodprob + gp)
pb = math.exp(badprob + bp) pb = math.exp(badprob + bp)
return pa - pb return pa - pb #+ random.uniform(-0.001,0.001)
def get_links(db,hostname,status,batch_size): def get_links(db,hostname,status,batch_size):
linkcol = db["links"] linkcol = db["links"]
# count downloaded links res = linkcol.find({"host":hostname,"status":status},limit=batch_size)
res = linkcol.aggregate([ links = []
{ "$match": { "status": {"$not":{"$in":["frontlink","backlink"]}},"host":hostname } }, for item in res:
{"$group":{"_id":None, links.append(item["url"])
"count":{"$count":{}}, print("Got {} {}".format(len(links),status))
} return links
},
])
links = set()
out = list(res)
if len(out) == 0:
return list()
if out[0]["count"] < 200:
#res = linkcol.find({"status":status,"host":hostname},{"url":1},limit=batch_size)
# get random links
res = linkcol.aggregate([
{ "$match": { "status": status,"host":hostname } },
{ "$sample": { "size": batch_size } }
])
for i,doc in enumerate(res):
#print(">>>>>" + status)
#print(doc);
links.add(doc["url"])
if i >= batch_size:
break
else:
cl = LinkClassifier()
cl.train(db,hostname)
res = linkcol.aggregate([
{ "$match": { "status": status,"host":hostname } },
{ "$sample": { "size": batch_size * 100 } }
])
outlinks = []
for i,doc in enumerate(res):
#print(">>>>>" + status)
#print(doc);
link = doc["url"]
outlinks.append((doc["url"],cl.classify(link)))
outlinks = sorted(outlinks, key=lambda x: x[1],reverse=True)
links = [l[0] for l in outlinks[0:batch_size]]
# todo remove very bad links from database
return list(links)
def fetch_sitemap_links(start_link): def fetch_sitemap_links(start_link):
@ -415,42 +390,53 @@ def fetch_sitemap_links(start_link):
navigation_links = trafilatura.sitemaps.sitemap_search(start_link,target_lang=LANGUAGE) navigation_links = trafilatura.sitemaps.sitemap_search(start_link,target_lang=LANGUAGE)
for link in navigation_links: for link in navigation_links:
out.append((link,"frontlink")) out.append((link,"frontlink"))
print("Fetched {} sitemap links".format(len(out)))
return out return out
def process_links(db,hostname,status,links=[],rules=None,batch_size=BATCHSIZE): def fetch_front_links(start_link,rules):
#print(links) start_link,hostname = courlan.check_url(start_link)
responses = fetch_pages(links) response = fetch_page(start_link)
#print(responses) extracted_links = extract_links([start_link],[response],hostname,rules,"frontlink")
extracted_pages = extract_pages(links,responses) print("Fetched {} frontlinks".format(len(extracted_links)))
#print(extracted_pages) return extracted_links
extracted_links = extract_links(links,responses,hostname,rules,status)
#print(extracted_links)
index_links(db,extracted_links)
index_pages(db,hostname,extracted_pages)
def link_summary(db,hostname): def link_summary(db,hostname):
linkcol = db["links"] linkcol = db["links"]
#res = linkcol.distinct("hostname",{"hostname":hostname}) #res = linkcol.distinct("hostname",{"hostname":hostname})
# count links
res = linkcol.aggregate([ res = linkcol.aggregate([
{"$match":{"host":hostname}}, {"$match":{"host":hostname}},
{"$group":{"_id":"$status","count":{"$sum":1}}}, {"$group":{"_id":"$status",
"count":{"$count":{}},
}
},
]) ])
badcount = 0 badcount = 0
goodcount = 0 goodcount = 0
out = ["good","frontlink","backlink"]
info = {} info = {}
crawled_count = 0
bad_crawl_count = 0
for item in res: for item in res:
if item["_id"] not in out: count = item["count"]
badcount += item["count"] st = item["_id"]
if item["_id"] == "good": print(st,count)
goodcount = item["count"] if st == "good":
info[item["_id"]] = item["count"] goodcount += count
good_prob = goodcount / (goodcount + badcount) if st != "frontlink" and st != "backlink":
crawled_count += count
if st != "good":
bad_crawl_count += count
info[st] = count
info["crawled_count"] = crawled_count
info["bad_crawl_count"] = bad_crawl_count
baclink_cout = 0
if "backlink" in info:
backlink_count = info["backlink"]
good_prob= 0
if crawled_count > 0:
good_prob = goodcount / crawled_count
info["good_prob"] = good_prob info["good_prob"] = good_prob
info["bad_documents"] = badcount
print(">>>Domain Content") print(">>>Domain Content")
contentcol = db["content"] contentcol = db["content"]
res = contentcol.aggregate([ res = contentcol.aggregate([
@ -465,34 +451,57 @@ def link_summary(db,hostname):
for item in res: for item in res:
text_size = item["text_size_sum"] text_size = item["text_size_sum"]
good_document_characters = 0 good_document_characters = 0
fetch_average_characters = 0
if goodcount > 0: if goodcount > 0:
good_document_characters = text_size / goodcount good_document_characters = text_size / goodcount
fetch_average_characters = text_size / (goodcount + badcount) fetch_average_characters = text_size / crawled_count
info["total_good_characters"] = text_size info["total_good_characters"] = text_size
info["average_good_characters"] = good_document_characters info["average_good_characters"] = good_document_characters
info["average_fetch_characters"] = fetch_average_characters info["average_fetch_characters"] = fetch_average_characters
domaincol = db["domain"] domaincol = db["domain"]
if goodcount + badcount > 100: domaincol.update_one({"host":hostname},{"$set":info},upsert=True)
cl = LinkClassifier() res = domaincol.find_one({"host":hostname})
cl.train(db,hostname) print(res)
res = linkcol.aggregate([
{ "$match": { "status": "backlink","host":hostname } }, def sample_links(db,hostname,status,batch_size):
{ "$sample": { "size": BATCHSIZE * 100 } } print("Getting backlinks")
]) linkcol = db["links"]
predicted_good = 0 res = linkcol.find({"host":hostname,"status": {"$not":{"$in":["frontlink","backlink"]}}})
predicted_bad = 0 cl = LinkClassifier()
crawled_links = list(res)
crawled_count = len(crawled_links)
prediction_accuracy = 0
if crawled_count > 200:
# train on crawled links
trainset,testset = split_train(crawled_links)
cl.train(trainset)
prediction_accuracy = cl.test(testset)
sample_set_size = 10000
res = linkcol.find({"host":hostname,"status": status},limit = sample_set_size)
sample_links = []
predicted_good = 0
for item in res:
for item in res: for item in res:
cll = cl.classify(item["url"]) cll = cl.classify(item["url"])
#cll += random.uniform(-0.1,0.1)
sample_links.append((item["url"],cll))
if cll > 0: if cll > 0:
predicted_good += 1 predicted_good += 1
else: # TODO frontlinks are not unique!
predicted_bad += 1 sample_links.sort(key=lambda x: x[1],reverse=True)
predicted_good_prob = 0 predicted_good_prob = 0
if predicted_good + predicted_bad > 0: if len(sample_links) > 0:
predicted_good_prob = predicted_good / (predicted_good + predicted_bad) predicted_good_prob = predicted_good / len(sample_links)
info["predicted_good_prob"] = predicted_good_prob domaincol = db["domain"]
info = {
"predicted_good_prob":predicted_good_prob,
"prediction_accuracy": prediction_accuracy,
"crawled_count": crawled_count,
}
print(info) print(info)
domaincol.update_one({"host":hostname},{"$set":info},upsert=True) domaincol.update_one({"host":hostname},{"$set":info})
links = [l[0] for l in sample_links[0:batch_size]]
return links
def domain_summary(db,hostname): def domain_summary(db,hostname):
linkcol = db["links"] linkcol = db["links"]
@ -517,13 +526,16 @@ def createdb():
linkcol.create_index("url",unique=True) linkcol.create_index("url",unique=True)
linkcol.create_index("host") linkcol.create_index("host")
contentcol = db["content"] contentcol = db["content"]
contentcol.create_index("url",unique=True) contentcol.create_index("url")
contentcol.create_index("text_md5",unique=True)
#contentcol.create_index({"paragraph_checksums":1}) #contentcol.create_index({"paragraph_checksums":1})
contentcol.create_index("host") contentcol.create_index("host")
htmlcol = db["html"] htmlcol = db["html"]
htmlcol.create_index("url",unique=True) htmlcol.create_index("url")
htmlcol.create_index("html_md5",unique=True)
domaincol = db["domains"] domaincol = db["domains"]
domaincol.create_index("host",unique=True) domaincol.create_index("host",unique=True)
domaincol.create_index("average_fetch_characters",unique=True)
@cli.command() @cli.command()
@click.argument("link") @click.argument("link")
@ -563,7 +575,12 @@ def classify(start_link):
db=myclient[DBNAME] db=myclient[DBNAME]
start_link,hostname = courlan.check_url(start_link) start_link,hostname = courlan.check_url(start_link)
cl = LinkClassifier() cl = LinkClassifier()
cl.train(db,hostname) linkcol = db["links"]
res = linkcol.find({"host":hostname,"status": {"$not":{"$in":["frontlink","backlink"]}}})
trainset, testset = split_train(res)
cl.train(trainset)
cl.test(testset)
@cli.command() @cli.command()
@click.argument("start_link") @click.argument("start_link")
@ -572,23 +589,25 @@ def visit(start_link):
db=myclient[DBNAME] db=myclient[DBNAME]
start_link,hostname = courlan.check_url(start_link) start_link,hostname = courlan.check_url(start_link)
batch_size = BATCHSIZE batch_size = BATCHSIZE
print("Getting frontlinks")
links = get_links(db,hostname,"frontlink",batch_size)
print(f"Got {len(links)} frontlinks")
if len(links) < batch_size:
print("Fetching sitemap links")
sitemap_links = fetch_sitemap_links(start_link)
index_links(db,sitemap_links)
links = get_links(db,hostname,"frontlink",batch_size)
links.insert(0,start_link)
if len(links) < batch_size:
back_links = get_links(db,hostname,"backlink",batch_size - len(links))
links += back_links
print("Processing links")
rules = fetch_robot(hostname) rules = fetch_robot(hostname)
responses = fetch_pages(links) # renew front links
sitemap_links = fetch_sitemap_links(start_link)
index_links(db,sitemap_links)
front_links = fetch_front_links(start_link,rules)
index_links(db,front_links)
# start crawling
# frontlinks first
links = sample_links(db,hostname,"frontlink",batch_size)
links.insert(0,start_link)
# then backlinks
if len(links) < batch_size:
back_links = sample_links(db,hostname,"backlink",batch_size - len(links))
links += back_links
# index results
print("Processing links")
responses = []
for link in links:
responses.append(fetch_page(link))
extracted_pages = extract_pages(links,responses) extracted_pages = extract_pages(links,responses)
extracted_links = extract_links(links,responses,hostname,rules,"backlink") extracted_links = extract_links(links,responses,hostname,rules,"backlink")
index_links(db,extracted_links) index_links(db,extracted_links)

15
mongo/mongoindexer.py
View File

@ -8,3 +8,18 @@ mycol = mydb["customers"]
mydict = {"text":"ahoj svet"} mydict = {"text":"ahoj svet"}
x = mycol.insert_one(mydict) x = mycol.insert_one(mydict)
def createdb():
myclient = pymongo.MongoClient(CONNECTION)
db=myclient[DBNAME]
linkcol = db["links"]
linkcol.create_index("url",unique=True)
linkcol.create_index("host")
contentcol = db["content"]
contentcol.create_index("url",unique=True)
#contentcol.create_index({"paragraph_checksums":1})
contentcol.create_index("host")
htmlcol = db["html"]
htmlcol.create_index("url")
domaincol = db["domains"]
domaincol.create_index("host",unique=True)

5
mongo/requirements.txt
View File

@ -1,2 +1,7 @@
trafilatura trafilatura
py3langid py3langid
courlan
pymongo
click
lxml
rq