409 lines
13 KiB
Python
Executable File
409 lines
13 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#! -*- coding: utf-8 -*-
|
|
import urllib.parse
|
|
import urllib.error
|
|
import os
|
|
import os.path
|
|
import re
|
|
import datetime
|
|
import time
|
|
import sys
|
|
import tempfile
|
|
import pprint
|
|
import bs4
|
|
|
|
import pycurl
|
|
import urllib.robotparser
|
|
import collections
|
|
import random
|
|
|
|
|
|
from websucker.parser import normalize_link,urlunparse
|
|
|
|
|
|
# Parses http refresh in header or on html meta
|
|
def get_refresh(ref,target_link):
|
|
refresh = None
|
|
tokens = ref.strip().split(";")
|
|
if len(tokens) > 1 and tokens[1].lower().startswith("url="):
|
|
refresh = urlunparse(normalize_link(
|
|
tokens[1][4:].strip("\'"), target_link))
|
|
return refresh
|
|
|
|
class Response:
|
|
def __init__(self,url,headers,status,content,redirects,link_status):
|
|
assert len(url) > 0
|
|
assert url[0] != "/"
|
|
self.url = url
|
|
self.status = status
|
|
self.content = content
|
|
self.headers = headers
|
|
self.redirects = redirects
|
|
self.visited_time = datetime.date.today()
|
|
self.bs = None
|
|
if content is not None and link_status == "good":
|
|
try:
|
|
self.bs = bs4.BeautifulSoup(content, "lxml")
|
|
except ValueError:
|
|
link_status = "bad_parse"
|
|
self.link_status = link_status
|
|
|
|
def __str__(self):
|
|
return "{} {} {}".format(self.url,self.get_canonical(),self.link_status)
|
|
|
|
def get_content(self):
|
|
if self.content is None:
|
|
print("NO CONTENT")
|
|
print(self.url,self.redirects)
|
|
return None
|
|
self.content.seek(0)
|
|
text = self.content.read()
|
|
out = str(text,encoding="utf8",errors="replace")
|
|
return out
|
|
|
|
|
|
# HMTL metarefresh redirect
|
|
def get_metarefresh(self):
|
|
if self.bs is None:
|
|
return None
|
|
metarefresh = None
|
|
t = self.bs.find_all("meta", attrs={"http-equiv": "refresh"})
|
|
canonical = self.get_canonical()
|
|
for tags in t:
|
|
if "content" in tags:
|
|
metarefresh = get_refresh(tags["content"],canonical)
|
|
if metarefresh is not None:
|
|
nl = normalize_link(metarefresh, canonical)
|
|
print("Metarefresh")
|
|
print(nl)
|
|
metarefresh = urlunparse(nl)
|
|
|
|
return metarefresh
|
|
|
|
def get_canonical(self):
|
|
r = None
|
|
last_link = self.url
|
|
if len(self.redirects) > 0:
|
|
last_link = self.redirects[-1]
|
|
if self.bs is not None:
|
|
l = self.bs.find("link", rel="canonical", href=True)
|
|
if l is not None:
|
|
r = urlunparse(normalize_link(l["href"], last_link))
|
|
if r is None:
|
|
r = last_link
|
|
r = urlunparse(normalize_link(r, last_link))
|
|
assert len(r) > 0
|
|
assert r[0] != "/"
|
|
return r
|
|
|
|
def get_redirects(self):
|
|
if len(self.redirects) <2 :
|
|
return []
|
|
return self.redirects[0:-1]
|
|
|
|
|
|
class Connection:
|
|
def __init__(self):
|
|
self.useragent = "Googlebot-News"
|
|
self.c = pycurl.Curl()
|
|
self.c.setopt(self.c.FOLLOWLOCATION, True)
|
|
# self.c.setopt(self.c.VERBOSE, True)
|
|
self.c.setopt(self.c.CONNECTTIMEOUT, 20)
|
|
self.c.setopt(self.c.TIMEOUT, 20)
|
|
self.c.setopt(self.c.FAILONERROR, True)
|
|
self.c.setopt(self.c.HTTPHEADER, [
|
|
'Accept: text/html', 'Accept-Charset: UTF-8'])
|
|
self.c.setopt(self.c.HEADERFUNCTION, self.header)
|
|
self.c.setopt(self.c.USERAGENT,self.useragent )
|
|
# #self.c.setopt(pycurl.COOKIEJAR, 'cookie.txt')
|
|
# #self.c.setopt(pycurl.COOKIEFILE, 'cookie.txt')
|
|
self.robots = {}
|
|
self.headers = {}
|
|
self.redirects = []
|
|
self.header_lines = []
|
|
self.status = 0
|
|
self.max_redirect = 4
|
|
|
|
# Zastavi spracovanie ak content nie je text
|
|
# zaznamena location a refresh
|
|
def header(self, data):
|
|
if len(data) == 0:
|
|
return None
|
|
l = str(data, encoding="utf8")
|
|
self.header_lines.append(l)
|
|
s = l.find(" ")
|
|
if s >= 1 and s < len(l):
|
|
key = l[0:s - 1]
|
|
value = l[s + 1:].rstrip()
|
|
self.headers[key] = value
|
|
kl = key.lower()
|
|
if kl == "refresh":
|
|
self.add_redirect(value)
|
|
elif kl == "location":
|
|
self.add_redirect(value)
|
|
elif kl == "content-type" and "text" not in value:
|
|
# Pycurl potom vyhodi 23, failed writing header
|
|
return 0
|
|
|
|
def crawl_delay(self,domain):
|
|
self.cache_robot(domain)
|
|
delay = 4
|
|
if domain in self.robots:
|
|
r = self.robots[domain]
|
|
if r is not None:
|
|
d = r.crawl_delay(self.useragent)
|
|
if d is not None:
|
|
delay = d
|
|
print("Waiting for {} s".format(delay))
|
|
time.sleep(delay)
|
|
|
|
def __del__(self):
|
|
self.c.close()
|
|
|
|
def close(self):
|
|
self.c.close()
|
|
|
|
def add_redirect(self,link):
|
|
last_link = self.url
|
|
if len(self.redirects) > 0:
|
|
last_link = self.redirects[-1]
|
|
v = urlunparse(normalize_link(link, last_link))
|
|
if v!=last_link and v not in set(self.redirects):
|
|
self.redirects.append(v)
|
|
|
|
"""
|
|
@returns content, link_status
|
|
@throws pycurl.error
|
|
"""
|
|
def _download(self, url):
|
|
print("Downloading " + url)
|
|
self.url = url
|
|
self.headers = {}
|
|
self.redirects = []
|
|
self.header_lines = []
|
|
self.status = 0
|
|
content = None
|
|
link_status = "bad_connection"
|
|
try:
|
|
self.headers = {}
|
|
del self.header_lines[:]
|
|
content = tempfile.SpooledTemporaryFile()
|
|
self.c.setopt(self.c.WRITEDATA, content)
|
|
self.c.setopt(self.c.URL, url)
|
|
self.c.perform()
|
|
self.status = self.c.getinfo(self.c.RESPONSE_CODE)
|
|
if self.status != 200:
|
|
link_status = "bad_httpcode"
|
|
elif "Content-Type" in self.headers and not self.headers["Content-Type"].startswith("text"):
|
|
link_status = "bad_type"
|
|
else:
|
|
link_status = "good"
|
|
content.seek(0)
|
|
except pycurl.error as e:
|
|
errno, message = e.args
|
|
content = None
|
|
self.status = self.c.getinfo(self.c.RESPONSE_CODE)
|
|
if errno == 23:
|
|
# 23 je zly content v header
|
|
link_status = "bad_type"
|
|
elif errno == 22:
|
|
link_status = "bad_httpcode"
|
|
elif errno == 28:
|
|
# 28 je connection timeout
|
|
link_status = "bad_connection"
|
|
elif errno == 60:
|
|
# 60 bad ssl certificate
|
|
link_status = "bad_connection"
|
|
elif errno == 56:
|
|
# 56 Connection reset by peer
|
|
link_status = "bad_connection"
|
|
elif errno == 16:
|
|
# 16 HTTP2
|
|
link_status = "bad_connection"
|
|
elif errno == 92:
|
|
# 92 HTTP2 not closed
|
|
link_status = "bad_connection"
|
|
elif errno == 6:
|
|
# 60 Unable to resolve dns
|
|
link_status = "bad_connection"
|
|
elif errno == 7:
|
|
# 7 Connection refused
|
|
link_status = "bad_connection"
|
|
else:
|
|
link_status = "bad_connection"
|
|
#raise e
|
|
except UnicodeDecodeError as e:
|
|
content = None
|
|
link_status = "bad_unicode"
|
|
except UnicodeEncodeError as e:
|
|
content = None
|
|
link_status = "bad_unicode"
|
|
sz = self.c.getinfo(self.c.SIZE_DOWNLOAD)
|
|
tt = self.c.getinfo(self.c.TOTAL_TIME)
|
|
print("{} Received {} bytes in {} s".format(self.status,sz,tt))
|
|
return content, link_status
|
|
|
|
# Throws pycurl.error
|
|
def html_download2(self, url):
|
|
dlink = url
|
|
responses = []
|
|
while len(responses) < 5:
|
|
nl = normalize_link(dlink)
|
|
url = urlunparse(nl)
|
|
assert url.startswith("http")
|
|
content, link_status = self._download(url)
|
|
response = Response(url,"\r\n".join(self.header_lines),self.status,content,self.redirects,link_status)
|
|
dlink = response.get_metarefresh()
|
|
responses.append(response)
|
|
if dlink is None:
|
|
break
|
|
return responses
|
|
|
|
def cache_robot(self,domain):
|
|
if domain not in self.robots:
|
|
roboturl = urlunparse(("https", domain, "robots.txt", ""))
|
|
try:
|
|
r = self._download(roboturl)
|
|
if r[1] == "good":
|
|
c = r[0].read()
|
|
lines = str(c, errors="ignore", encoding="utf8").split("\n")
|
|
self.robots[domain] = urllib.robotparser.RobotFileParser()
|
|
self.robots[domain].parse(lines)
|
|
else:
|
|
self.robots[domain] = None
|
|
except pycurl.error as err:
|
|
print(err)
|
|
|
|
def is_robot_good(self, url):
|
|
schema, domain, path, query = normalize_link(url)
|
|
self.cache_robot(domain)
|
|
res = True
|
|
if domain in self.robots and self.robots[domain] is not None:
|
|
res = self.robots[domain].can_fetch("Agent", url)
|
|
return res
|
|
|
|
|
|
|
|
def parse_and_index(work_link,parser,responses,db):
|
|
"""
|
|
Take all responses from work link, parse and store in db
|
|
@param work_link - final link from downloader
|
|
@param parser to use
|
|
@param responses from the downloader
|
|
@param db
|
|
"""
|
|
target_link = work_link
|
|
links = []
|
|
if len(responses) > 0:
|
|
db.index_responses(work_link,responses)
|
|
lr = responses[-1]
|
|
if lr.bs is not None:
|
|
target_link = lr.get_canonical()
|
|
parsed = parser.full_extract(lr.content,lr.bs,target_link)
|
|
db.index_content(target_link,parsed)
|
|
links = parsed.link_set
|
|
return target_link,links
|
|
|
|
def visit_sitemap(domain,connection,parser,db):
|
|
"""
|
|
get links from sitemap of the domain
|
|
"""
|
|
link = "http://" + domain
|
|
print("Sitemap visit: " + link)
|
|
responses = connection.html_download2(link)
|
|
if len(responses) == 0:
|
|
return False
|
|
lr = responses[-1]
|
|
if lr.bs is None:
|
|
return False
|
|
|
|
target_link,outlinks = parse_and_index(link,parser,responses,db)
|
|
if len(outlinks) > 0:
|
|
db.index_follow_links(parser,outlinks,connection)
|
|
return True
|
|
|
|
|
|
def visit_links(links,connection,parser,db,is_online=True):
|
|
"""
|
|
if the site is not online, then just check links
|
|
"""
|
|
outlinks = []
|
|
junklinks = []
|
|
badrobotlinks = []
|
|
for work_link in links:
|
|
responses = []
|
|
if not parser.is_link_good(work_link):
|
|
db.update_link_status(work_link,"bad_link")
|
|
elif is_online and not connection.is_robot_good(work_link):
|
|
db.update_link_status(work_link,"bad_robot")
|
|
elif is_online:
|
|
responses = connection.html_download2(work_link)
|
|
target_link,links = parse_and_index(work_link,parser,responses,db)
|
|
nl = normalize_link(target_link)
|
|
connection.crawl_delay(nl[1])
|
|
outlinks += links
|
|
if len(outlinks) > 0:
|
|
db.index_follow_links(parser,outlinks,connection)
|
|
|
|
def visit_domain(domain,parser,db):
|
|
"""
|
|
One visit of the domain
|
|
|
|
1.Get links from the frontpage,
|
|
2. visit links and extract new links
|
|
3. get new links to visit
|
|
4. repeat visit for parser.crawl_rounds
|
|
"""
|
|
c = Connection()
|
|
p = parser
|
|
# Get links from frontpage
|
|
# TODO Sitemap
|
|
is_online = False
|
|
if parser.is_domain_good(domain):
|
|
# Is domain online?
|
|
is_online = visit_sitemap(domain,c,parser,db)
|
|
if is_online:
|
|
for i in range(p.crawl_rounds):
|
|
# Visit links from frontpage
|
|
links = db.get_visit_links(domain,p.recent_links,p.old_links,p.random_links)
|
|
visit_links(links,c,p,db,is_online)
|
|
db.check_domain(domain)
|
|
else:
|
|
db.check_domain(domain)
|
|
return True
|
|
|
|
def process_domains(domains,visit,parser,db,queue):
|
|
"""
|
|
Visit all domains in list.
|
|
if queue is true, then queue domain instead immediate visit
|
|
"""
|
|
print("Websucker Agenda>>")
|
|
random.shuffle(domains)
|
|
for domain in domains:
|
|
assert len(domain[0]) > 1
|
|
print(domain)
|
|
if queue is not None:
|
|
print("Queuing:")
|
|
for domain in domains:
|
|
print(domain)
|
|
queue.put(domain[0])
|
|
queue.close()
|
|
if visit:
|
|
print("Visiting:")
|
|
for domain in domains:
|
|
print(domain)
|
|
visit_domain(domain[0],parser,db)
|
|
|
|
def work_domains(parser,db,queue):
|
|
"""
|
|
Poll the queue and visit
|
|
"""
|
|
while True:
|
|
print("Waiting for a new job:")
|
|
job = queue.reserve()
|
|
domain = job.body
|
|
queue.bury(job)
|
|
print("Visiting:")
|
|
visit_domain(domain,parser,db)
|
|
queue.delete(job)
|