websucker-pip/mongo/mongocwarler.py

import pymongo
import trafilatura
import trafilatura.feeds
import trafilatura.sitemaps
import trafilatura.spider
import sys
import courlan


def calculate_checksums(text):
    """
    @return fingerprints of a paragraphs in text. Paragraphs are separated by a blank line
    """
    checksums = []
    sizes = []
    hval = 0
    hsz = 0
    sz = 0
    for c in text:
        cv = ord(c)
        sz += 1
        if cv > 64:
            hval += (hval << 3) + cv
            zv = hval >> 31
            hval &= 0x7fffffff
            hval += zv
            hsz += 1
        if c == "\n" and hsz > 0:
            if hsz > 100:
                checksums.append(hval)
                sizes.append(sz)
            sz = 0
            hsz = 0
    if hsz > 100:
        checksums.append(hval)
        sizes.append(sz)
    return checksums, sizes

def filter_links(links,language="sk"):
    out = set()
    for link in links:
        r = courlan.check_url(link,strict=True,language=language)
        if r is None:
            continue
        llink,ldomain = r
        print(llink,ldomain)
        # domain rules
        if not ldomain.endswith("sk"):
            print("bad domain")
            continue
        if courlan.is_not_crawlable(llink):
            print("not crawlable")
            continue
        out.add(llink)
    return out

def get_link_doc(link,status="frontlink"):
    r  = courlan.check_url(link)
    assert r is not None
    link,host = r
    domain = courlan.extract_domain(link)
    return {"url":link,"host":host,"domain":domain,"status":status}

def generic_visit(domain):
    known_links = set(get_visited_links(domain))
    visit_links = []
    visit_links = trafilatura.find_feed_urls(domain)
    if visit_links is None:
        visit_links = trafilatura.sitemap_search(domain)
    if visit_links is None:
        visit_links = trafilatura.focused_crawler(dommain,known_links=known_links)


def fetch_pages(link_batch):
    htmls  = []
    print(link_batch)
    print("zzzzzzzzzz")
    for link in link_batch:
        print("fetching:::::")
        print(link)
        htmls.append(trafilatura.fetch_url(link))
    return htmls

def fetch_front_links(navigation_links):
    start_link = navigation_links[0]
    known_links = navigation_links[1:]
    visit_links,known_links = trafilatura.spider.focused_crawler(start_link,known_links=known_links)
    filtered_links = filter_links(visit_links)
    return filtered_links

def extract_pages(link_batch,responses):
    out = []
    for link,response in zip(link_batch,responses):
        doc = None
        assert link is not None
        html = trafilatura.util.decode_response(response.data) 
        if html is not None:
            doc = trafilatura.bare_extraction(html,url=link,with_metadata=True,include_formatting=True,target_language="sk")
        out.append((link,response.url,html,doc))
    return out



def index_pages(db,domain,extracted_pages):
    extracted_links = set()
    linkcol = db["links"]
    htmlcol = db["html"]
    contentcol = db["content"]
    for link,html,doc in extracted_pages:
        state = "good"
        if html is None:
            state = "html_error"
        elif doc is None:
            state = "content_error"
        if html is not None:
            htmlcol.insert_one({"url":link,"html":html})
        if doc is not None:
            print(doc)
            checksums,sizes = calculate_checksums(doc["text"])
            doc["paragraph_checksums"] = checksums
            doc["paragraph_sizes"] = sizes
            if "links" in doc:
                extracted_links.union(doc["links"])
                del doc["links"]
            contentcol.insert_one(doc)
        doc = get_link_doc(link,state)
        linkcol.replace_one({"url":link},doc,upsert=True)
        filtered_links = filter_links(extracted_links)
        for llink in filtered_links:
            doc = get_link_doc(link,"backlink")
            if courlan.is_external(link,domain):
                doc["status"]= "frontlink"
            elif courlan.is_navigation(link):
                doc["status"] = "navigation"
            linkcol.insert_one(doc)

def get_links(db,domain,status,batch_size=100):
    linkcol = db["links"]
    res  = linkcol.find({"status":status,"hostname":domain},limit=batch_size)
    front_links = []
    for doc in res:
        front_links.append(doc["url"])
    return filter_links(front_links)


def index_front_links(db,filtered_links):
    linkcol = db["links"]
    for link in filtered_links:
        linkcol.insert_one(get_link_doc(link,"frontlink"))


    #visit_links = trafilatura.feeds.find_feed_urls(domain)
    #visit_links = trafilatura.sitemaps.sitemap_search(domain)
    #print(visit_links)
    #for link in visit_links:
    #    content = trafilatura.fetch_url(link,decode=True)
    #    document = trafilatura.bare_extraction(content)
    #    print(content)

def simple_visit(start_link):
    start_link,domain = courlan.check_url(start_link)
    myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/")
    db=myclient["crawler"]
    navigation_links =[start_link]
    navigation_links += get_links(db,"navigation",domain)
    new_front_links = fetch_front_links(navigation_links)
    print("NEW FRONT LINKS")
    print(new_front_links)
    index_front_links(db,new_front_links)
    front_links = get_links(db,"frontlink",domain)
    print("NEW VISIT LINKS")
    visit_links = front_links
    print(visit_links)
    responses = fetch_pages(visit_links)
    extracted_pages = extract_pages(visit_links,responses)
    index_pages(db,domain,extracted_pages)

simple_visit(sys.argv[1])