.dockerignore

@@ -1,2 +0,0 @@
-venv
-websucker.egg-info

Dockerfile

@@ -1,16 +1,14 @@
-FROM python:3.8-slim
+FROM python:3.8.0-alpine
 
-RUN apt-get update && apt-get install -y git curl  libcurl4-openssl-dev build-essential vim libssl-dev
+RUN apk add --update --no-cache git curl curl-dev vim py3-lxml gcc make libxml2-dev libxslt-dev libc-dev
 
+RUN addgroup -S appgroup -g 1000 && \
+    adduser -u 1000 -S appuser -G appgroup
 
-RUN addgroup appgroup && \
-    adduser  appuser  && adduser appuser appgroup
+RUN mkdir /app
 
-RUN mkdir /app /src
-
-ADD requirements.txt /src
-RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip install -r /src/requirements.txt
-ADD .  /src
-RUN pip install /src
+ADD requirements.txt /
+RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip install -r /requirements.txt
+RUN pip install https://git.kemt.fei.tuke.sk/dano/websucker-pip/archive/master.zip
 WORKDIR /app
 ENTRYPOINT ["websuck"]

websucker/agent.py

@@ -430,7 +430,7 @@ def visit_sitemap(domain,connection,parser,db):
     return True
 
 
-def visit_links(links,connection,parser,db,is_online=True):
+def visit_links(links,connection,parser,db,is_online):
     """
     if the site is not online, then just check links
     """

websucker/cli.py

@@ -132,7 +132,7 @@ def start(ctx, link):
     p = ctx.obj["parser"]
     c = Connection()
     visit_links([link],c,p,db)
-    #db.check_domain(domain)
+    db.check_domain(domain)
 
 @cli.command(help="Continue crawling of seen links from a domain")
 @click.pass_context