merge

Dockerfile

@@ -1,16 +1,16 @@
-FROM python:3.8
+FROM debian:11
 
-RUN apt-get update && apt-get install -y git curl  libcurl4-openssl-dev build-essential vim libssl-dev python-pip
+RUN apt-get update && apt-get install -y git curl  libcurl4-openssl-dev build-essential vim libssl-dev python2 python-dev wget python3 python3-pip
-
+RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python2 get-pip.py
-RUN pip2 install cqlsh
+RUN python2 -m pip install cqlsh
 
 RUN addgroup appgroup && \
     adduser  appuser  && adduser appuser appgroup
 
 RUN mkdir /app /src
 ADD requirements.txt  /src/
-RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip install -r /src/requirements.txt
+RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip3 install -r /src/requirements.txt
 WORKDIR /src
 ADD . /src
-RUN python /src/setup.py install
+RUN pip3 install /src/
 WORKDIR /app

build-docker.sh

@@ -0,0 +1 @@
+docker build . -t dr.kemt.fei.tuke.sk/websucker:dev

websucker/cli.py

@@ -158,6 +158,13 @@ def check(ctx,domain):
     res = db.check_domain(domain)
     print(res)
 
+@cli.command(help="Export domain as JSON doc per line")
+@click.pass_context
+@click.argument("domain")
+def tojson(ctx,domain):
+    db = create_database_from_context(ctx)
+    db.export_domain(domain)
+
 @cli.command(help="Print daily report")
 @click.pass_context
 def report(ctx):

websucker/db.py

@@ -1,5 +1,6 @@
 import cassandra
 import cassandra.cluster
+import cassandra.query
 from cassandra.auth import PlainTextAuthProvider
 import random
 import os
@@ -8,6 +9,7 @@ import datetime
 from websucker.parser import normalize_link,urlunparse
 import collections
 import math
+import json
 
 VERSION = "sucker6"
 
@@ -543,6 +545,11 @@ INSERT INTO content(
                     result.append((d,0))
         return result
 
+    def export_domain(self,domain):
+        rows = self.session.execute("SELECT JSON * from content WHERE domain_name=%s",(domain,))
+        for row in rows:
+            print(row[0])
+    
     def get_visit_links(self,domain,recent_count,old_count,random_count):
         dblinks = []
         rows = self.session.execute("SELECT url_schema,url_path,url_query,update_time FROM links Where domain_name=%s AND link_status='seen'",(domain,))