websucker-pip/websucker/agent.py

#!/usr/bin/env python
#! -*- coding: utf-8 -*-
import urllib.parse
import urllib.error
import os
import os.path
import re
import datetime
import time
import sys
import tempfile
import pprint
import bs4

import pycurl
import urllib.robotparser
import collections
import random


from websucker.parser import normalize_link,urlunparse


# Parses http refresh in header or on html meta
def get_refresh(ref,target_link):
    refresh = None
    tokens = ref.strip().split(";")
    if len(tokens) > 1 and tokens[1].lower().startswith("url="):
        refresh = urlunparse(normalize_link(
            tokens[1][4:].strip("\'"), target_link))
    return refresh

class Response:
    def __init__(self,url,headers,status,content,redirects,link_status):
        assert len(url) > 0
        assert url[0] != "/"
        self.url = url
        self.status = status
        self.content = content
        self.headers = headers
        self.redirects = redirects
        self.visited_time = datetime.date.today()
        self.bs = None
        if content is not None and link_status == "good":
            try:
                self.bs = bs4.BeautifulSoup(content, "lxml")
            except ValueError:
                link_status = "bad_parse"
        self.link_status = link_status

    def __str__(self):
        return "{} {} {}".format(self.url,self.get_canonical(),self.link_status)

    def get_content(self):
        if self.content is None:
            print("NO CONTENT")
            print(self.url,self.redirects)
            return None
        self.content.seek(0)
        text = self.content.read()
        out = str(text,encoding="utf8",errors="replace")
        return out


    # HMTL metarefresh redirect
    def get_metarefresh(self):
        if self.bs is None:
            return None
        metarefresh = None
        t = self.bs.find_all("meta", attrs={"http-equiv": "refresh"})
        canonical = self.get_canonical()
        for tags in t:
            if "content" in tags:
                metarefresh = get_refresh(tags["content"],canonical)
        if metarefresh is not None:
            nl = normalize_link(metarefresh, canonical)
            print("Metarefresh")
            print(nl)
            metarefresh = urlunparse(nl)

        return metarefresh

    def get_canonical(self):
        r = None
        last_link = self.url
        if len(self.redirects) > 0:
            last_link = self.redirects[-1]
        if self.bs is not None:
            l = self.bs.find("link", rel="canonical", href=True)
            if l is not None:
                r = urlunparse(normalize_link(l["href"], last_link))
        if r is None:
            r = last_link
        r = urlunparse(normalize_link(r, last_link))
        assert len(r) > 0
        assert r[0] != "/"
        return r

    def get_redirects(self):
        if len(self.redirects) <2 :
            return []
        return self.redirects[0:-1]


class Connection:
    def __init__(self):
        self.useragent = "Googlebot-News"
        self.c = pycurl.Curl()
        self.c.setopt(self.c.FOLLOWLOCATION, True)
#       self.c.setopt(self.c.VERBOSE, True)
        self.c.setopt(self.c.CONNECTTIMEOUT, 20)
        self.c.setopt(self.c.TIMEOUT, 20)
        self.c.setopt(self.c.FAILONERROR, True)
        self.c.setopt(self.c.HTTPHEADER, [
                      'Accept: text/html', 'Accept-Charset: UTF-8'])
        self.c.setopt(self.c.HEADERFUNCTION, self.header)
        self.c.setopt(self.c.USERAGENT,self.useragent )
#        #self.c.setopt(pycurl.COOKIEJAR, 'cookie.txt')
#        #self.c.setopt(pycurl.COOKIEFILE, 'cookie.txt')
        self.robots = {}
        self.headers = {}
        self.redirects = []
        self.header_lines = []
        self.status = 0
        self.max_redirect = 4

    # Zastavi spracovanie ak content nie je text
    # zaznamena location a refresh
    def header(self, data):
        if len(data) == 0:
            return None
        l = str(data, encoding="utf8")
        self.header_lines.append(l)
        s = l.find(" ")
        if s >= 1 and s < len(l):
            key = l[0:s - 1]
            value = l[s + 1:].rstrip()
            self.headers[key] = value
            kl = key.lower()
            if kl == "refresh":
                self.add_redirect(value)
            elif kl == "location":
                self.add_redirect(value)
            elif kl == "content-type" and "text" not in value:
                # Pycurl potom vyhodi 23, failed writing header
                return 0

    def crawl_delay(self,domain):
        self.cache_robot(domain)
        delay = 4
        if domain in self.robots:
            r = self.robots[domain]
            if r is not None:
                d = r.crawl_delay(self.useragent)
                if d is not None:
                    delay = d
        print("Waiting for {} s".format(delay))
        time.sleep(delay)

    def __del__(self):
        self.c.close()

    def close(self):
        self.c.close()
    
    def add_redirect(self,link):
        last_link = self.url
        if len(self.redirects) > 0:
            last_link = self.redirects[-1]
        v = urlunparse(normalize_link(link, last_link))
        if v!=last_link and v not in set(self.redirects):
            self.redirects.append(v)

    """
    @returns content, link_status 
    @throws pycurl.error
    """
    def _download(self, url):
        print("Downloading " + url)
        self.url = url
        self.headers = {}
        self.redirects = []
        self.header_lines = []
        self.status = 0
        content = None
        link_status = "bad_connection"
        try:
            self.headers = {}
            del self.header_lines[:]
            content = tempfile.SpooledTemporaryFile()
            self.c.setopt(self.c.WRITEDATA, content)
            self.c.setopt(self.c.URL, url)
            self.c.perform()
            self.status = self.c.getinfo(self.c.RESPONSE_CODE)
            if self.status != 200:
                link_status = "bad_httpcode"
            elif "Content-Type" in self.headers and not self.headers["Content-Type"].startswith("text"):
                link_status = "bad_type"
            else:
                link_status = "good"
                content.seek(0)
        except pycurl.error as e:
            errno, message = e.args
            content = None
            self.status = self.c.getinfo(self.c.RESPONSE_CODE)
            if errno == 23:
                # 23 je zly content v header
                link_status = "bad_type"
            elif errno == 22:
                link_status = "bad_httpcode"
            elif errno == 28:
                # 28 je connection timeout
                link_status = "bad_connection"
            elif errno == 60:
                # 60 bad ssl certificate
                link_status = "bad_connection"
            elif errno == 56:
                # 56 Connection reset by peer
                link_status = "bad_connection"
            elif errno == 16:
                # 16 HTTP2
                link_status = "bad_connection"
            elif errno == 92:
                # 92 HTTP2 not closed
                link_status = "bad_connection"
            elif errno == 6:
                # 60 Unable to resolve dns
                link_status = "bad_connection"
            elif errno == 7:
                # 7 Connection refused
                link_status = "bad_connection"
            else:
                link_status = "bad_connection"
                #raise e
        except UnicodeDecodeError as e:
            content = None
            link_status = "bad_unicode"
        except UnicodeEncodeError as e:
            content = None
            link_status = "bad_unicode"
        sz = self.c.getinfo(self.c.SIZE_DOWNLOAD)
        tt = self.c.getinfo(self.c.TOTAL_TIME)
        print("{} Received {} bytes in {} s".format(self.status,sz,tt))
        return content, link_status

    # Throws pycurl.error
    def html_download2(self, url):
        dlink = url
        responses = []
        while len(responses) < 5:
            nl = normalize_link(dlink)
            url = urlunparse(nl)
            assert url.startswith("http")
            content, link_status = self._download(url)
            response = Response(url,"\r\n".join(self.header_lines),self.status,content,self.redirects,link_status)
            dlink = response.get_metarefresh()
            responses.append(response)
            if dlink is None:
                break
        return responses

    def cache_robot(self,domain):
        if domain not in self.robots:
            roboturl = urlunparse(("https", domain, "robots.txt", ""))
            try:
                r = self._download(roboturl)
                if r[1] == "good":
                    c = r[0].read()
                    lines = str(c, errors="ignore", encoding="utf8").split("\n")
                    self.robots[domain] = urllib.robotparser.RobotFileParser()
                    self.robots[domain].parse(lines)
                else:
                    self.robots[domain] = None
            except pycurl.error as err:
                print(err)

    def is_robot_good(self, url):
        schema, domain, path, query = normalize_link(url)
        self.cache_robot(domain)
        res = True
        if domain in self.robots and self.robots[domain] is not None:
            res = self.robots[domain].can_fetch("Agent", url)
        return res



def parse_and_index(work_link,parser,responses,db):
    """
    Take all responses from work link, parse and store in db
    @param work_link - final link from downloader
    @param parser to use
    @param responses from the downloader
    @param db 
    """
    target_link = work_link
    links = []
    if len(responses) > 0:
        db.index_responses(work_link,responses)
        lr = responses[-1]
        if lr.bs is not None:
            target_link = lr.get_canonical()
            parsed = parser.full_extract(lr.content,lr.bs,target_link)
            db.index_content(target_link,parsed)
            links = parsed.link_set
    return target_link,links

def visit_sitemap(domain,connection,parser,db):
    """
    get links from sitemap of the domain
    """
    link = "http://" + domain
    print("Sitemap visit: " + link)
    responses = connection.html_download2(link)
    if len(responses) == 0:
        return False
    lr = responses[-1]
    if lr.bs is None:
        return False

    target_link,outlinks = parse_and_index(link,parser,responses,db)
    if len(outlinks) > 0:
        db.index_follow_links(parser,outlinks,connection)
    return True


def visit_links(links,connection,parser,db,is_online=True):
    """
    if the site is not online, then just check links
    """
    outlinks = []
    junklinks = []
    badrobotlinks = []
    for work_link in links:
        responses = []
        if not parser.is_link_good(work_link):
            db.update_link_status(work_link,"bad_link")
        elif is_online and not connection.is_robot_good(work_link):
            db.update_link_status(work_link,"bad_robot")
        elif is_online:
            responses = connection.html_download2(work_link)
            target_link,links = parse_and_index(work_link,parser,responses,db)
            nl = normalize_link(target_link)
            connection.crawl_delay(nl[1])
            outlinks += links
    if len(outlinks) > 0:
        db.index_follow_links(parser,outlinks,connection)

def visit_domain(domain,parser,db):
    """
    One visit of the domain

    1.Get links from the frontpage,
    2. visit links and extract new links
    3. get new links to visit
    4. repeat visit for parser.crawl_rounds
    """
    c = Connection()
    p = parser
    # Get links from frontpage
    # TODO Sitemap
    is_online = False
    if parser.is_domain_good(domain):
        # Is domain online?
        is_online = visit_sitemap(domain,c,parser,db) 
    if is_online:
        for i in range(p.crawl_rounds):
            # Visit links from frontpage
            links = db.get_visit_links(domain,p.recent_links,p.old_links,p.random_links)
            visit_links(links,c,p,db,is_online)
            db.check_domain(domain)
    else:
        db.check_domain(domain)
    return True

def process_domains(domains,visit,parser,db,queue):
    """
    Visit all domains in list.
    if queue is true, then queue domain instead immediate visit
    """
    print("Websucker Agenda>>")
    random.shuffle(domains)
    for domain in domains:
        assert len(domain[0]) > 1
        print(domain)
    if queue is not None:
        print("Queuing:")
        for domain in domains:
            print(domain)
            queue.put(domain[0])
        queue.close()
    if visit:
        print("Visiting:")
        for domain in domains:
            print(domain)
            visit_domain(domain[0],parser,db)

def work_domains(parser,db,queue):
    """
    Poll the queue and visit
    """
    while True:
        print("Waiting for a new job:")
        job = queue.reserve()
        domain = job.body
        queue.bury(job)
        print("Visiting:")
        visit_domain(domain,parser,db)
        queue.delete(job)