websucker/dump.py

@@ -4,7 +4,6 @@ import cassandra.query
 import json
 import datetime
 import sys
-from urllib.parse import urlparse
 
 cassandra_host = sys.argv[1]
 cassandra_port = sys.argv[2]
@@ -17,30 +16,9 @@ select_documents = "select json * from content"
 with cluster.connect(keyspace) as session:
     #session.row_factory = cassandra.query.dict_factory
     select_html = session.prepare("select json * from html where day=? and domain_name=? LIMIT 1")
-    select_link = session.prepare("select link_status from links where domain_name=? and url_path=? and url_query=? LIMIT 1")
     rows = session.execute(select_documents)
     for row in rows:
         doc = json.loads(row["[json]"])
-        target_link = doc["target_link"]
-        parsed_link = urlparse(target_link)
-        netloc = parsed_link[1].strip().lower()
-        path = parsed_link[2].strip()
-        # strip leading /
-        if len(path) > 1 and path[0] == "/":
-            path = path[1:]
-        query = parsed_link[4]
-        lrows = session.execute(select_link,(netloc,path,query))
-        status = None
-        for l in lrows:
-            status = str(l["link_status"])
-            break
-        #assert status is not None
-        if status is None:
-            continue
-        #print(status)
-        # skip bad links
-        if not status == "good":
-            continue
         dt = doc["update_time"]
         d = dt.split()[0]
         hrows = session.execute(select_html,(d,doc["domain_name"]))
@@ -49,6 +27,4 @@ with cluster.connect(keyspace) as session:
             html = json.loads(h["[json]"])
             break
         doc["html_data"] = html
-        del doc["links"]
-        #print(parsed_link)
         print(json.dumps(doc))