zz

mongo/mongocwarler.py

@@ -7,12 +7,15 @@ import trafilatura.utils
 import sys
 import courlan
 import urllib
+from datetime import datetime
+import click
 
 LANGUAGE="sk"
 DOMAIN = "sk"
 BATCHSIZE=10
 MINFILESIZE=300
-MAXFILESIZE=1000000
+MAXFILESIZE=10000000
+MINTEXTSIZE=200
 
 def calculate_checksums(text):
     """
@@ -83,7 +86,7 @@ def get_link_doc(link,status="frontlink"):
     assert r is not None
     link,host = r
     domain = courlan.extract_domain(link)
-    return {"url":link,"host":host,"domain":domain,"status":status}
+    return {"url":link,"host":host,"domain":domain,"status":status,"created_at":datetime.utcnow()}
 
 def generic_visit(domain):
     known_links = set(get_visited_links(domain))
@@ -147,6 +150,10 @@ def extract_pages(link_batch,responses):
         assert original_link is not None
         if html is not None:
             doc = trafilatura.bare_extraction(html,url=final_link,with_metadata=True,include_formatting=True,target_language=LANGUAGE)
+            if doc is not None:
+                if not "text" in doc or len(doc["text"]) < MINTEXTSIZE:
+                    # text too small
+                    doc = None
         out.append((original_link,final_link,html,doc))
     return out
 
@@ -167,10 +174,12 @@ def index_pages(db,domain,extracted_pages):
             linkcol.insert_one(get_link_doc(final_link,state))
             state = "redirect"
         linkcol.update_one({"url":original_link},{"$set":{"status":state}})
-        if html is not None:
-            htmlcol.insert_one({"url":final_link,"html":html})
         if doc is not None:
+            if html is not None:
+                htmlcol.insert_one({"url":final_link,"html":html,"html_size":len(html),"created_at":datetime.utcnow()})
             checksums,sizes = calculate_checksums(doc["text"])
+            doc["created_at"] = datetime.utcnow()
+            doc["text_size"] = len(doc["text"])
             doc["paragraph_checksums"] = checksums
             doc["paragraph_sizes"] = sizes
             # todo extract links
@@ -231,6 +240,30 @@ def process_links(db,domain,status,links=[],rules=None,batch_size=BATCHSIZE):
     index_links(db,extracted_links)
     index_pages(db,domain,extracted_pages)
 
+
+def link_summary(db,domain):
+    linkcol = db["links"]
+    #res = linkcol.distinct("domain",{"hostname":domain})
+
+    res = linkcol.aggregate([
+        {"$match":{"host":domain}},
+        {"$group":{"_id":"$status","count":{"$sum":1}}},
+    ])
+    for item in res:
+        print(item)
+
+def create_indices(db):
+    linkcol = db["links"]
+    linkcol.create_index({"url":1},{"name":"url"})
+    linkcol.create_index({"host":1,"status":1},{"name":"hostname_status"})
+    contentcol = db["content"]
+    contentcol.create_index({"url":1})
+    contentcol.create_index({"paragraph_checksums":1})
+    contentcol.create_index({"domain":1})
+    htmlcol = db["html"]
+    htmlcol.create_index({"url":1})
+
+@click.command()
 def simple_visit(start_link=None):
     start_link,domain = courlan.check_url(start_link)
     myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/")
@@ -249,25 +282,3 @@ def simple_visit(start_link=None):
         front_links = get_links(db,domain,"backlink",bl)
     process_links(db,domain,"backlink",links,rules=rules)
     link_summary(db,domain)
-
-def link_summary(db,domain):
-    linkcol = db["links"]
-    res = linkcol.aggregate([
-        {"$match":{"hostname":domain}},
-        {"$group":{"_id":{"status":domain},"count":{"$count":1}}},
-    ])
-    print(res)
-
-def create_indices(db):
-    linkcol = db["links"]
-    linkcol.create_index({"url":1},{"name":"url"})
-    linkcol.create_index({"hostname":1,"status":1},{"name":"hostname_status"})
-    contentcol = db["content"]
-    contentcol.create_index({"url":1})
-    contentcol.create_index({"paragraph_checksums":1})
-    contentcol.create_index({"domain":1})
-    htmlcol = db["html"]
-    htmlcol.create_index({"url":1})
-
-
-simple_visit(sys.argv[1])