mongo/mongocrawler.py

@@ -739,13 +739,6 @@ def crawl_summary():
     for item in res:
         values = [str(item[x]) for x in headers]
         print("\t".join(values))
-    contentcol = db["content"]
-    res = contentcol.aggregate([
-        {"$group":{"_id":None,total_text_size:{"$sum":"$text_size"}}}
-    ])
-    print(">>>>> Total text size in content")
-    for item in res:
-        print(res)
 
 
 def import_html():

mongo/submitdomains.sh

@@ -1,4 +0,0 @@
-DOMAINS=`cut -f 1 -d ";" domains.txt | shuf`
-for DOM in $DOMAINS ; do
-    echo rq enqueue mongocrawler.visit hostname=$DOM
-done