z

mongo/mongocwarler.py

@@ -36,24 +36,29 @@ def calculate_checksums(text):
         sizes.append(sz)
     return checksums, sizes
 
-
-
-
 def filter_links(links):
     out = set()
     for link in links:
         r = courlan.check_url(link)
         if r is None:
             continue
-        link,ldomain = r
+        llink,ldomain = r
+        print(llink,ldomain)
         # domain rules
         if not ldomain.endswith("sk"):
+            print("bad domain")
             continue
-        if not courlan.is_crawlable(link):
+        if courlan.is_not_crawlable(llink):
+            print("not crawlable")
             continue
-        out.add(link)
+        out.add(llink)
     return out
 
+def get_link_doc(link,status="frontlink"):
+    r  = courlan.check_url(link)
+    assert r is not None
+    link,domain = r
+    return {"url":link,"domain":domain,"status":status}
 
 def generic_visit(domain):
     known_links = set(get_visited_links(domain))
@@ -65,27 +70,25 @@ def generic_visit(domain):
         visit_links = trafilatura.focused_crawler(dommain,known_links=known_links)
 
 
-def get_new_links(domain):
-    known_links = []
-    # get seed links
-    # get new links
-    out_links = trafilatura.spider.focused_crawler(domain,known_links=known_links)
-    filtered_links = filter_links(out_links)
-    print(res)
-    return filtered_links
 
-def fetch_links(link_batch):
+def fetch_pages(link_batch):
     htmls  = []
     print(link_batch)
     print("zzzzzzzzzz")
     for link in link_batch:
+        print("fetching:::::")
         print(link)
         rr = trafilatura.fetch_url(link,decode=True)
         htmls.append(rr)
     return htmls
 
+def fetch_front_links(start_link):
+    known_links = []
+    visit_links,known_links = trafilatura.spider.focused_crawler(start_link,known_links=known_links)
+    filtered_links = filter_links(visit_links)
+    return filtered_links
 
-def extract_links(link_batch,htmls):
+def extract_pages(link_batch,htmls):
     out = []
     for link,html in zip(link_batch,htmls):
         doc = None
@@ -95,7 +98,9 @@ def extract_links(link_batch,htmls):
         out.append((link,html,doc))
     return out
 
-def index_pages(db,extracted_pages,domain):
+
+
+def index_pages(db,domain,extracted_pages):
     extracted_links = set()
     linkcol = db["links"]
     htmlcol = db["html"]
@@ -117,21 +122,33 @@ def index_pages(db,extracted_pages,domain):
                 extracted_links.union(doc["links"])
                 del doc["links"]
             contentcol.insert_one(doc)
-        linkcol.replace_one({"url":link},{"url":link,"status":state},upsert=True)
+        doc = get_link_doc(link,state)
+        linkcol.replace_one({"url":link},doc,upsert=True)
         filtered_links = filter_links(extracted_links)
-        for link in filtered_links:
-            linkcol.insert_one({"url":link,"status":"backlink"},upsert=True)
+        for llink in filtered_links:
+            doc = get_link_doc(link,"backlink")
+            linkcol.insert_one(doc,upsert=True)
+
+def get_front_links(db,domain,batch_size=100):
+    linkcol = db["links"]
+    res  = linkcol.find({"status":"frontlink","domain":domain},limit=batch_size)
+    front_links = []
+    for doc in res:
+        front_links.append(doc["url"])
+    return filter_links(front_links)
+
+def get_back_links(db,domain,batch_size=100):
+    linkcol = db["links"]
+    frontlinks = linkcol.find({"status":"backlink","domain":domain},limit=batch_size)
+    return front_links
 
 
-def simple_visit(start_link):
-    known_links = []
-    visit_links,known_links = trafilatura.spider.focused_crawler(start_link,known_links=known_links)
-    htmls = fetch_links(visit_links)
-    extracted_links = extract_links(visit_links,htmls)
-    myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/")
+def index_front_links(db,filtered_links):
+    linkcol = db["links"]
+    for link in filtered_links:
+        linkcol.insert_one(get_link_doc(link,"frontlink"))
+
 
-    db = myclient["mydatabase"]
-    index_pages(db,extracted_links,start_link)
     #visit_links = trafilatura.feeds.find_feed_urls(domain)
     #visit_links = trafilatura.sitemaps.sitemap_search(domain)
     #print(visit_links)
@@ -140,4 +157,19 @@ def simple_visit(start_link):
     #    document = trafilatura.bare_extraction(content)
     #    print(content)
 
+def simple_visit(start_link):
+    start_link,domain = courlan.check_url(start_link)
+    myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/")
+    db=myclient["crawler"]
+    new_front_links = fetch_front_links(start_link)
+    print("NEW FRONT LINKS")
+    print(new_front_links)
+    index_front_links(db,new_front_links)
+    visit_links = get_front_links(db,domain)
+    print("NEW VISIT LINKS")
+    print(visit_links)
+    htmls = fetch_pages(visit_links)
+    extracted_pages = extract_pages(visit_links,htmls)
+    index_pages(db,domain,extracted_pages)
+
 simple_visit(sys.argv[1])