zz
This commit is contained in:
parent
9e36952563
commit
e5a7fc1e99
11
Dockerfile
Normal file
11
Dockerfile
Normal file
@ -0,0 +1,11 @@
|
||||
FROM python:3.8.0-alpine
|
||||
|
||||
RUN apk add --update --no-cache git curl curl-dev vim py3-lxml gcc make libxml2-dev libxslt-dev libc-dev
|
||||
|
||||
RUN addgroup -S appgroup -g 1000 && \
|
||||
adduser -u 1000 -S appuser -G appgroup
|
||||
|
||||
ADD requirements.txt /
|
||||
RUN CASS_DRIVER_BUILD_CONCURRENCY=4 pip install -r /requirements.txt
|
||||
RUN pip install https://git.kemt.fei.tuke.sk/dano/websucker-pip/archive/master.zip
|
||||
ENTRYPOINT ["websuck"]
|
@ -1,6 +1,8 @@
|
||||
BeautifulSoup4
|
||||
justext
|
||||
cassandra-driver
|
||||
python-dateutil
|
||||
click
|
||||
pycurl
|
||||
lz4
|
||||
lxml
|
||||
cassandra-driver
|
||||
|
5
setup.py
5
setup.py
@ -32,11 +32,12 @@ setuptools.setup(
|
||||
install_requires=[
|
||||
"BeautifulSoup4",
|
||||
"justext",
|
||||
"cassandra-driver",
|
||||
"python-dateutil",
|
||||
"click",
|
||||
"pycurl",
|
||||
"greenstalk"
|
||||
"greenstalk",
|
||||
"lz4",
|
||||
"cassandra-driver",
|
||||
],
|
||||
|
||||
)
|
||||
|
@ -60,7 +60,7 @@ class Response:
|
||||
|
||||
# HMTL metarefresh redirect
|
||||
def get_metarefresh(self):
|
||||
if self.content is None:
|
||||
if self.bs is None:
|
||||
return None
|
||||
metarefresh = None
|
||||
t = self.bs.find_all("meta", attrs={"http-equiv": "refresh"})
|
||||
@ -132,11 +132,12 @@ class Connection:
|
||||
key = l[0:s - 1]
|
||||
value = l[s + 1:].rstrip()
|
||||
self.headers[key] = value
|
||||
if key.lower() == "refresh":
|
||||
kl = key.lower()
|
||||
if kl == "refresh":
|
||||
self.add_redirect(value)
|
||||
elif key.lower() == "location":
|
||||
elif kl == "location":
|
||||
self.add_redirect(value)
|
||||
elif key == "Content-Type" and "text" not in value:
|
||||
elif kl == "content-type" and "text" not in value:
|
||||
# Pycurl potom vyhodi 23, failed writing header
|
||||
return 0
|
||||
|
||||
@ -209,12 +210,18 @@ class Connection:
|
||||
elif errno == 60:
|
||||
# 60 bad ssl certificate
|
||||
link_status = "bad_connection"
|
||||
elif errno == 56:
|
||||
# 56 Connection reset by peer
|
||||
link_status = "bad_connection"
|
||||
elif errno == 16:
|
||||
# 16 HTTP2
|
||||
link_status = "bad_connection"
|
||||
elif errno == 6:
|
||||
# 60 Unable to resolve dns
|
||||
link_status = "bad_connection"
|
||||
elif errno == 7:
|
||||
# 7 Connection refused
|
||||
link_status = "bad_connection"
|
||||
else:
|
||||
raise e
|
||||
except UnicodeDecodeError as e:
|
||||
@ -300,7 +307,7 @@ class ParsedDocument:
|
||||
self.paragraph_sizes = pszs
|
||||
if bs is None:
|
||||
return
|
||||
self.tags,self.authors,self.title,self.article_publilshed_time, self.description,self.section = self.parser.extract_og(bs)
|
||||
self.tags,self.authors,self.title,self.article_publilshed_time, self.description,self.section = self.parser.extract_meta(bs)
|
||||
|
||||
# Extrakcia linkov zo stranky
|
||||
base = self.work_link
|
||||
|
@ -26,18 +26,18 @@ def create_queue_from_context(ctx):
|
||||
@click.option("--beanstalkd-tube",metavar="BEANSTALKD_TUBE",help="beanstalkd keyspace (if defined, value read from BEANSTALKD_TUBE env variable)",envvar="BEANSTALKD_TUBE",default="websucker",show_default=True)
|
||||
@click.option("--beanstalkd-host",metavar="BEANSTALKD_HOST",help="beanstalkd host (if defined, value read from beanstalkd_HOST env variable)",envvar="BEANSTALKD_HOST",default="127.0.0.1",show_default=True)
|
||||
@click.option("--beanstalkd-port",metavar="BEANSTALKD_PORT",help="beanstalkd port (if defined, value read from BEANSTALKD_PORT env variable)",envvar="BEANSTALKD_PORT",default=11300,show_default=True)
|
||||
@click.option("--justext-language",metavar="JUSTEXT_LANGUAGE",help="Target language (if defined, value read from JUSTEXT_LANGUAGE env variable)",envvar="JUSTEXT_LANGUAGE",default="English",show_default=True)
|
||||
@click.option("--parser",metavar="file_name",help="zzz")
|
||||
@click.option("--visit",is_flag=True)
|
||||
@click.option("--queue",is_flag=True)
|
||||
def cli(ctx,cassandra_keyspace,cassandra_host,cassandra_port,beanstalkd_tube,beanstalkd_host,beanstalkd_port,justext_language,parser,visit,queue):
|
||||
def cli(ctx,cassandra_keyspace,cassandra_host,cassandra_port,beanstalkd_tube,beanstalkd_host,beanstalkd_port,parser,visit,queue):
|
||||
ctx.ensure_object(dict)
|
||||
p = BaseParser()
|
||||
p.justext_language = justext_language
|
||||
|
||||
suckerfile = os.getcwd() + "/Suckerfile.py"
|
||||
if os.path.isfile(suckerfile):
|
||||
parser = suckerfile
|
||||
if parser is not None:
|
||||
assert os.path.isfile(parser)
|
||||
else:
|
||||
suckerfile = os.getcwd() + "/Suckerfile.py"
|
||||
if os.path.isfile(suckerfile):
|
||||
parser = suckerfile
|
||||
if parser is not None:
|
||||
p = load_parser(parser)
|
||||
assert p is not None
|
||||
@ -130,12 +130,22 @@ def check(ctx,domain):
|
||||
def report(ctx):
|
||||
db = create_database_from_context(ctx)
|
||||
db.daily_report()
|
||||
if ctx.obj["queue"]:
|
||||
try:
|
||||
q = create_queue_from_context(ctx)
|
||||
stats = q.stats_tube(ctx.obj["beanstalkd_tube"])
|
||||
buried = stats["current-jobs-buried"]
|
||||
ready = stats["current-jobs-buried"]
|
||||
ready = stats["current-jobs-ready"]
|
||||
print("{} ready jobs, {} burried jobs".format(ready,buried))
|
||||
except Error as err:
|
||||
print(err)
|
||||
|
||||
@cli.command(help="Database summary")
|
||||
@click.pass_context
|
||||
def summary(ctx):
|
||||
db = create_database_from_context(ctx)
|
||||
p = ctx.obj["parser"]
|
||||
db.summary(p)
|
||||
|
||||
@cli.command(help="Print keyspace schema")
|
||||
def schema():
|
||||
schema = get_schema()
|
||||
|
@ -160,7 +160,43 @@ INSERT INTO content(
|
||||
VERSION,
|
||||
)
|
||||
self.session.execute(self.index_response_insert_html,d)
|
||||
|
||||
|
||||
def summary(self,parser):
|
||||
gs = 0
|
||||
cs = 0
|
||||
fc = 0
|
||||
vd = 0
|
||||
ud = 0
|
||||
sl = 0
|
||||
fd = 0
|
||||
jd = 0
|
||||
rows = self.session.execute("SELECT domain_name,good_size,content_size, fetched_count,seen_count FROM domain_quality PER PARTITION LIMIT 1")
|
||||
for row in rows:
|
||||
if not parser.is_domain_good(row[0]):
|
||||
jd += 1
|
||||
if row[1] is not None:
|
||||
gs += row[1]
|
||||
if row[2] is not None:
|
||||
cs += row[2]
|
||||
if row[3] is not None:
|
||||
fc += row[3]
|
||||
if row[4] is not None:
|
||||
sl += row[4]
|
||||
if row[3] is None or row[3] == 0:
|
||||
ud += 1
|
||||
else:
|
||||
vd += 1
|
||||
if row[4] is None or row[4] == 0:
|
||||
fd += 1
|
||||
print("Good characters: {}".format(gs))
|
||||
print("Fetched characters: {}".format(cs))
|
||||
print("Fetched documents: {}".format(fc))
|
||||
print("Visited domains: {}".format(vd))
|
||||
print("Unvisited domains: {}".format(ud))
|
||||
print("Junk domains: {}".format(jd))
|
||||
print("New links : {}".format(sl))
|
||||
print("Finished domains : {}".format(fd))
|
||||
|
||||
def daily_report(self):
|
||||
#rows = self.session.execute(self.daily_links_select)
|
||||
rows = self.session.execute("SELECT domain_name,count(link_status) FROM daily_links WHERE day=toDate(now()) GROUP BY day,domain_name")
|
||||
|
@ -237,7 +237,7 @@ class BaseParser:
|
||||
# Extracts matainformation from html
|
||||
# First it looks for name, content in meta tags
|
||||
# then it looks for opengraph
|
||||
def extract_og(self, bs):
|
||||
def extract_meta(self, bs):
|
||||
tags = set()
|
||||
authors = set()
|
||||
title = ""
|
||||
|
Loading…
Reference in New Issue
Block a user