websucker-pip/websucker/schema.py

from cassandra.cqlengine import columns
from cassandra.cqlengine.models import Model
from cassandra.cqlengine.management import sync_table

class Links(Model):
    __table_name__ = "links"
    domain_name = columns.Text(primary_key=True)
    url_path = columns.Text(primary_key=True)
    url_query = columns.Text(primary_key=True)
    url_schema = columns.Text()
    redirect_target = columns.Text()
    link_status = columns.Text(index=True)
    link_originality = columns.Float()
    body_size = columns.Integer()
    update_time = columns.DateTime()


class DailyLinks(Model):
    __table_name__ = "daily_links"
    day = columns.Date(primary_key=True)
    domain_name = columns.Text(primary_key=True)
    link_status = columns.Text(primary_key=True)
    url_path = columns.Text(primary_key=True)
    url_query = columns.Text(primary_key=True)
    link_originality = columns.Float()
    body_size = columns.Integer()
    update_time = columns.DateTime()


class DomainQuality(Model):
    __table_name__ = "domain_quality"
    domain_name = columns.Text(primary_key=True)
    day = columns.Date(primary_key=True)
    seen_count = columns.Integer()
    good_size = columns.Integer()
    good_count = columns.Integer()
    good_probability = columns.Float()
    good_originality = columns.Float()
    average_good_characters = columns.Float()
    content_size = columns.Integer()
    content_count = columns.Integer()
    content_probability = columns.Float()
    content_originality = columns.Float()
    average_content_characters = columns.Float()
    fetched_count = columns.Integer()
    average_fetched_good_characters = columns.Float()
    gain_ratio = columns.Float()
    update_time = columns.DateTime(static=True) #TIMESTAMP STATIC ,
#) WITH CLUSTERING ORDER BY (day DESC);

class Content(Model):
    __table_name__ = "content"
    domain_name = columns.Text(primary_key=True)
    target_link = columns.Text(primary_key=True)
    agent_version = columns.Text()
    title = columns.Text()
    links = columns.Set(value_type=columns.Text)
    authors = columns.Set(value_type=columns.Text)
    tags = columns.Set(value_type=columns.Text)
    description = columns.Text()
    section = columns.Text()
    article_published_time = columns.Text()
    text_date = columns.Text()
    body = columns.Text()
    body_size = columns.Integer()
    update_time = columns.DateTime()
#    PRIMARY KEY(domain_name,target_link),

class ParagraphChecksums(Model):
    __table_name__ = "paragraph_checksums"
    checksum = columns.BigInt(primary_key=True)
    url_hash = columns.BigInt(primary_key=True)

class Html(Model):
    __table_name__ = "html"
    day = columns.Date(primary_key=True)
    domain_name = columns.Text(primary_key=True)
    source_link = columns.Text(primary_key=True)
    target_link = columns.Text()
    redirect_links = columns.List(value_type=columns.Text)
    status =  columns.Integer()
    content = columns.Text()
    headers = columns.Text()
    agent_version = columns.Text()
    update_time = columns.DateTime()

class DomainConnections(Model):
    __table_name__ = "domain_connections"
    domain_name = columns.Text(primary_key=True)
    linked_domain = columns.Text(primary_key=True)

def create_database(keyspace,session):
    sync_table(Links)
    sync_table(DailyLinks)
    sync_table(DomainQuality)
    sync_table(Content)
    sync_table(ParagraphChecksums)
    sync_table(Html)
    sync_table(DomainConnections)