zz

mongo/Dockerfile

@@ -4,4 +4,4 @@ COPY requirements.txt /app
 RUN pip install -r /app/requirements.txt
 COPY *.py /app
 WORKDIR /app
-ENTRYPOINT ["python", "./mongocrawler.py"]
+ENTRYPOINT ["rq", "worker"]

mongo/docker-compose.yaml

@@ -1,5 +1,9 @@
 version: "3.0"
 services:
+  redis:
+    image: redis
+    ports:
+     - 6379:6379
   mongo:
     image: mongo
     environment:

mongo/mongocrawler.py

@@ -591,8 +591,6 @@ def visit(start_link):
     batch_size = BATCHSIZE
     rules = fetch_robot(hostname)
     # renew front links
-    sitemap_links = fetch_sitemap_links(start_link)
-    index_links(db,sitemap_links)
     front_links = fetch_front_links(start_link,rules)
     index_links(db,front_links)
     # start crawling