Compare commits
No commits in common. "1e7ac17d903a136674146a0f9a2ea94cca93d2d4" and "d37bafb666f1e2984304e52afd9dc85cc6553896" have entirely different histories.
1e7ac17d90
...
d37bafb666
@ -12,7 +12,7 @@ import os
|
||||
|
||||
|
||||
def create_database_from_context(ctx):
|
||||
return Data(ctx.obj["cassandra_keyspace"],ctx.obj["cassandra_host"],ctx.obj["cassandra_port"],ctx.obj["cassandra_username"],ctx.obj["cassandra_password"])
|
||||
return Data(ctx.obj["cassandra_keyspace"],ctx.obj["cassandra_host"],ctx.obj["cassandra_port"])
|
||||
|
||||
def create_queue_from_context(ctx):
|
||||
return greenstalk.Client((ctx.obj["beanstalkd_host"],ctx.obj["beanstalkd_port"]),use=ctx.obj["beanstalkd_tube"],watch=ctx.obj["beanstalkd_tube"],encoding="utf8")
|
||||
@ -23,15 +23,13 @@ def create_queue_from_context(ctx):
|
||||
@click.option("--cassandra-keyspace",metavar="CASSANDRA_KEYSPACE",help="cassandra keyspace (if defined, value read from CASSANDRA_KEYSPACE env variable)",envvar="CASSANDRA_KEYSPACE",default="websucker",show_default=True)
|
||||
@click.option("--cassandra-host",metavar="CASSANDRA_HOST",help="cassandra host (if defined, value read from CASSANDRA_HOST env variable)",envvar="CASSANDRA_HOST",default="127.0.0.1",show_default=True)
|
||||
@click.option("--cassandra-port",metavar="CASSANDRA_PORT",help="cassandra port (if defined, value read from CASSANDRA_PORT env variable)",envvar="CASSANDRA_PORT",default=9042,show_default=True)
|
||||
@click.option("--cassandra-username",metavar="CASSANDRA_USERNAME",help="cassandra username (if defined, value read from CASSANDRA_USERNAME env variable)",envvar="CASSANDRA_USERNAME")
|
||||
@click.option("--cassandra-password",metavar="CASSANDRA_PASSWORD",help="cassandra password (if defined, value read from CASSANDRA_PASSWORD env variable)",envvar="CASSANDRA_PASSWORD")
|
||||
@click.option("--beanstalkd-tube",metavar="BEANSTALKD_TUBE",help="beanstalkd keyspace (if defined, value read from BEANSTALKD_TUBE env variable)",envvar="BEANSTALKD_TUBE",default="websucker",show_default=True)
|
||||
@click.option("--beanstalkd-host",metavar="BEANSTALKD_HOST",help="beanstalkd host (if defined, value read from beanstalkd_HOST env variable)",envvar="BEANSTALKD_HOST",default="127.0.0.1",show_default=True)
|
||||
@click.option("--beanstalkd-port",metavar="BEANSTALKD_PORT",help="beanstalkd port (if defined, value read from BEANSTALKD_PORT env variable)",envvar="BEANSTALKD_PORT",default=11300,show_default=True)
|
||||
@click.option("--parser",metavar="file_name",help="zzz")
|
||||
@click.option("--visit",is_flag=True)
|
||||
@click.option("--queue",is_flag=True)
|
||||
def cli(ctx,cassandra_keyspace,cassandra_host,cassandra_port,cassandra_username,cassandra_password,beanstalkd_tube,beanstalkd_host,beanstalkd_port,parser,visit,queue):
|
||||
def cli(ctx,cassandra_keyspace,cassandra_host,cassandra_port,beanstalkd_tube,beanstalkd_host,beanstalkd_port,parser,visit,queue):
|
||||
ctx.ensure_object(dict)
|
||||
p = BaseParser()
|
||||
if parser is not None:
|
||||
@ -46,8 +44,6 @@ def cli(ctx,cassandra_keyspace,cassandra_host,cassandra_port,cassandra_username,
|
||||
ctx.obj["parser"] = p
|
||||
ctx.obj["cassandra_host"] = cassandra_host
|
||||
ctx.obj["cassandra_port"] = cassandra_port
|
||||
ctx.obj["cassandra_username"] = cassandra_username
|
||||
ctx.obj["cassandra_password"] = cassandra_password
|
||||
ctx.obj["cassandra_keyspace"] = cassandra_keyspace
|
||||
ctx.obj["beanstalkd_host"] = beanstalkd_host
|
||||
ctx.obj["beanstalkd_port"] = beanstalkd_port
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
import cassandra
|
||||
import cassandra.cluster
|
||||
from cassandra.auth import PlainTextAuthProvider
|
||||
import random
|
||||
import os
|
||||
import pkg_resources
|
||||
@ -21,15 +20,12 @@ class Data:
|
||||
"""
|
||||
Database of text documents
|
||||
"""
|
||||
def __init__(self,keyspace="websucker",cassandra_host="127.0.0.1",cassandra_port=9042,username=None,password=None):
|
||||
def __init__(self,keyspace="websucker",cassandra_host="127.0.0.1",cassandra_port=9042):
|
||||
print("Database {}@{}:{}".format(keyspace,cassandra_host, cassandra_port))
|
||||
auth_provider = None
|
||||
if username is not None and password is not None:
|
||||
auth_provider = PlainTextAuthProvider(username=username, password=password)
|
||||
# execution profile
|
||||
ep = cassandra.cluster.ExecutionProfile(request_timeout=240.0)
|
||||
profiles = {cassandra.cluster.EXEC_PROFILE_DEFAULT:ep}
|
||||
self.cluster = cassandra.cluster.Cluster([cassandra_host],port=cassandra_port,execution_profiles=profiles,auth_provider=auth_provider)
|
||||
self.cluster = cassandra.cluster.Cluster([cassandra_host],port=cassandra_port,execution_profiles=profiles)
|
||||
self.session = self.cluster.connect(keyspace)
|
||||
|
||||
self.check_document_select_query = self.session.prepare("SELECT count(url_hash) FROM paragraph_checksums WHERE checksum=?" )
|
||||
|
||||
@ -1,93 +0,0 @@
|
||||
from cassandra.cqlengine import columns
|
||||
from cassandra.cqlengine.models import Model
|
||||
|
||||
class Links(Model):
|
||||
domain_name = column.Text(primary_key=True)
|
||||
url_path = column.Text(primary_key=True)
|
||||
url_query = column.Text(primary_key=True)
|
||||
url_schema = column.Text()
|
||||
redirect_target = column.Text()
|
||||
link_status = column.Text()
|
||||
link_originality = column.Float()
|
||||
body_size = column.Integer()
|
||||
update_time = column.DateTime()
|
||||
|
||||
|
||||
class DailyLinks(Model):
|
||||
day = column.Integer(primary_key=True)
|
||||
domain_name = column.Text(primary_key=True)
|
||||
url_path = column.Text(primary_key=True)
|
||||
url_query = column.Text(primary_key=True)
|
||||
url_schema = column.Text()
|
||||
redirect_target = column.Text()
|
||||
link_status = column.Text()
|
||||
link_originality = column.Float()
|
||||
body_size = column.Integer()
|
||||
update_time = column.DateTime()
|
||||
|
||||
|
||||
CREATE TABLE domain_quality (
|
||||
domain_name TEXT,
|
||||
day DATE,
|
||||
seen_count INT,
|
||||
good_size INT,
|
||||
good_count INT,
|
||||
good_probability FLOAT,
|
||||
good_originality FLOAT,
|
||||
average_good_characters FLOAT,
|
||||
content_size INT,
|
||||
content_count INT,
|
||||
content_probability FLOAT,
|
||||
content_originality FLOAT,
|
||||
average_content_characters FLOAT,
|
||||
fetched_count INT,
|
||||
average_fetched_good_characters FLOAT,
|
||||
gain_ratio FLOAT,
|
||||
update_time TIMESTAMP STATIC ,
|
||||
PRIMARY KEY(domain_name,day)
|
||||
) WITH CLUSTERING ORDER BY (day DESC);
|
||||
|
||||
|
||||
CREATE TABLE content (
|
||||
domain_name TEXT,
|
||||
target_link TEXT,
|
||||
agent_version TEXT,
|
||||
title TEXT,
|
||||
links SET<TEXT>,
|
||||
authors SET<TEXT>,
|
||||
tags SET<TEXT>,
|
||||
description TEXT,
|
||||
section TEXT,
|
||||
article_published_time TEXT,
|
||||
text_date TEXT,
|
||||
body TEXT,
|
||||
body_size INT,
|
||||
update_time TIMESTAMP,
|
||||
PRIMARY KEY(domain_name,target_link),
|
||||
);
|
||||
|
||||
CREATE TABLE paragraph_checksums (
|
||||
checksum BIGINT,
|
||||
url_hash BIGINT,
|
||||
PRIMARY KEY(checksum,url_hash),
|
||||
);
|
||||
|
||||
CREATE TABLE html (
|
||||
day DATE,
|
||||
domain_name TEXT,
|
||||
source_link TEXT,
|
||||
target_link TEXT,
|
||||
redirect_links LIST<TEXT>,
|
||||
status INT,
|
||||
content TEXT,
|
||||
headers TEXT,
|
||||
agent_version TEXT,
|
||||
update_time TIMESTAMP,
|
||||
PRIMARY KEY(day,domain_name,source_link)
|
||||
);
|
||||
|
||||
CREATE TABLE domain_connections (
|
||||
domain_name TEXT,
|
||||
linked_domain TEXT,
|
||||
PRIMARY KEY (domain_name,linked_domain)
|
||||
);
|
||||
Loading…
Reference in New Issue
Block a user