Dockerfile

@@ -1,16 +1,16 @@
-FROM debian:11
+FROM python:3.8
 
-RUN apt-get update && apt-get install -y git curl  libcurl4-openssl-dev build-essential vim libssl-dev python2 python-dev wget python3 python3-pip
+RUN apt-get update && apt-get install -y git curl  libcurl4-openssl-dev build-essential vim libssl-dev python-pip
-RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python2 get-pip.py
+
-RUN python2 -m pip install cqlsh
+RUN pip2 install cqlsh
 
 RUN addgroup appgroup && \
     adduser  appuser  && adduser appuser appgroup
 
 RUN mkdir /app /src
 ADD requirements.txt  /src/
-RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip3 install -r /src/requirements.txt
+RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip install -r /src/requirements.txt
 WORKDIR /src
 ADD . /src
-RUN pip3 install /src/
+RUN python /src/setup.py install
 WORKDIR /app

build-docker.sh

@@ -1 +0,0 @@
-docker build . -t dr.kemt.fei.tuke.sk/websucker:dev

websucker/cli.py

@@ -158,13 +158,6 @@ def check(ctx,domain):
     res = db.check_domain(domain)
     print(res)
 
-@cli.command(help="Export domain as JSON doc per line")
-@click.pass_context
-@click.argument("domain")
-def tojson(ctx,domain):
-    db = create_database_from_context(ctx)
-    db.export_domain(domain)
-
 @cli.command(help="Print daily report")
 @click.pass_context
 def report(ctx):

websucker/db.py

@@ -1,6 +1,5 @@
 import cassandra
 import cassandra.cluster
-import cassandra.query
 from cassandra.auth import PlainTextAuthProvider
 import random
 import os
@@ -9,7 +8,6 @@ import datetime
 from websucker.parser import normalize_link,urlunparse
 import collections
 import math
-import json
 
 VERSION = "sucker6"
 
@@ -545,11 +543,6 @@ INSERT INTO content(
                     result.append((d,0))
         return result
     
-    def export_domain(self,domain):
-        rows = self.session.execute("SELECT JSON * from content WHERE domain_name=%s",(domain,))
-        for row in rows:
-            print(row[0])
-    
     def get_visit_links(self,domain,recent_count,old_count,random_count):
         dblinks = []
         rows = self.session.execute("SELECT url_schema,url_path,url_query,update_time FROM links Where domain_name=%s AND link_status='seen'",(domain,))