zz

mongo/cli.py

@@ -9,32 +9,42 @@ REDIS_URL= os.getenv("REDIS_URL","redis://localhost:6379/")
 QUEUES=os.getenv("QUEUES","high,default,low")
 
 @click.group()
-def cli():
+@click.option("--dbname",default=mongocrawler.DBNAME,help="database to use")
+def cli(dbname):
+    DBNAME=dbname
     pass
 
 @cli.command()
 def createdb():
     mongocrawler.createdb()
 
+@cli.command()
+def dropdb():
+    mongocrawler.dropdb()
+
 @cli.command()
 @click.argument("link")
 def parseurl(link):
+    """ Parse document on link for debug """
     mongocrawler.parseurl(link)
 
 @cli.command()
 @click.argument("link")
 def externaldomains(link):
+    """ Extract external domains from link """
     mongocrawler.externaldomains(link)
 
 @cli.command()
 @click.argument("start_link")
 def classify(start_link):
+    """ domain to  to classify links for debug """
     mongocrawler.classify(start_link)
 
 @cli.command()
-@click.argument("hostname",help="Hostname to crawl")
+@click.argument("hostname")
 @click.option("--filter_content",default=True,help="Filter content")
 def visit(hostname,filter_content=True):
+    """ Hostname to crawl """
     mongocrawler.visit(hostname,filter_content=filter_content)
 
 @cli.command()

mongo/mongocrawler.py

@@ -87,6 +87,7 @@ def get_bs_links(link,html):
             print(err)
             pass
     return links
+
 def split_train(res):
     trainset = []
     testset = []
@@ -99,7 +100,9 @@ def split_train(res):
 
 def calculate_checksums(text):
     """
-    @return fingerprints of a paragraphs in text. Paragraphs are separated by a blank line
+    Paragraph separation must be compatible with text extraction. Are paragraphs separated with a blank line or a white line?
+
+    @return fingerprints of a paragraphs in text. Paragraphs are separated by a new line. 
     """
     checksums = []
     sizes = []
@@ -153,11 +156,11 @@ def is_link_good(link):
     return llink
 
 def get_link_doc(link:str,status="frontlink")->dict:
-    r  = courlan.check_url(link)
-    assert r is not None
-    link,host = r
-    domain = courlan.extract_domain(link)
-    return {"url":link,"host":host,"domain":domain,"status":status,"created_at":dat.utcnow()}
+    parsed = urllib.parse.urlparse(courlan.normalize_url(link))
+    url = urllib.parse.urlunparse(parsed)
+    tokens = parsed.netloc.split(".")
+    domain =  tokens[-2] + "." + tokens[-1]
+    return {"url":link,"host":parsed.netloc,"domain":domain,"status":status,"created_at":dat.utcnow()}
 
 
 def fetch_page(link:str)->(str,str):
@@ -600,6 +603,12 @@ def domain_summary(db,hostname):
     for item in res:
         print(item)
 
+def dropdb():
+    myclient = pymongo.MongoClient(CONNECTION)
+    print("write name of database to drop")
+    dbname = sys.stdin.readline().strip()
+    myclient.drop_database(dbname)
+
 
 def createdb():
     myclient = pymongo.MongoClient(CONNECTION)
@@ -617,7 +626,7 @@ def createdb():
     htmlcol.create_index("html_md5",unique=True)
     domaincol = db["domains"]
     domaincol.create_index("host",unique=True)
-    domaincol.create_index(("average_fetch_characters",pymongo.DESCENDING))
+    domaincol.create_index([("average_fetch_characters",pymongo.DESCENDING)])
     batchcol = db["batches"]
     batchcol.create_index("host")
     batchcol.create_index("created_at")
@@ -745,9 +754,11 @@ def import_html():
     for l in sys.stdin:
         hdoc = json.loads(l)
         url = hdoc["url"]
+        # beautifusoup is to unify encoding
         html = BeautifulSoup(binascii.a2b_qp(hdoc["quoted_html"])).prettify()
         doc = extract_page(url,html)
         if doc is not None:
+            print("------=====-")
             print(doc)
             status = index_page(db,url,url,html,doc)
             print(status)