zz
This commit is contained in:
commit
a7d048c952
14
mongo/cli.py
14
mongo/cli.py
@ -9,32 +9,42 @@ REDIS_URL= os.getenv("REDIS_URL","redis://localhost:6379/")
|
||||
QUEUES=os.getenv("QUEUES","high,default,low")
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
@click.option("--dbname",default=mongocrawler.DBNAME,help="database to use")
|
||||
def cli(dbname):
|
||||
DBNAME=dbname
|
||||
pass
|
||||
|
||||
@cli.command()
|
||||
def createdb():
|
||||
mongocrawler.createdb()
|
||||
|
||||
@cli.command()
|
||||
def dropdb():
|
||||
mongocrawler.dropdb()
|
||||
|
||||
@cli.command()
|
||||
@click.argument("link")
|
||||
def parseurl(link):
|
||||
""" Parse document on link for debug """
|
||||
mongocrawler.parseurl(link)
|
||||
|
||||
@cli.command()
|
||||
@click.argument("link")
|
||||
def externaldomains(link):
|
||||
""" Extract external domains from link """
|
||||
mongocrawler.externaldomains(link)
|
||||
|
||||
@cli.command()
|
||||
@click.argument("start_link")
|
||||
def classify(start_link):
|
||||
""" domain to to classify links for debug """
|
||||
mongocrawler.classify(start_link)
|
||||
|
||||
@cli.command()
|
||||
@click.argument("hostname",help="Hostname to crawl")
|
||||
@click.argument("hostname")
|
||||
@click.option("--filter_content",default=True,help="Filter content")
|
||||
def visit(hostname,filter_content=True):
|
||||
""" Hostname to crawl """
|
||||
mongocrawler.visit(hostname,filter_content=filter_content)
|
||||
|
||||
@cli.command()
|
||||
|
@ -87,6 +87,7 @@ def get_bs_links(link,html):
|
||||
print(err)
|
||||
pass
|
||||
return links
|
||||
|
||||
def split_train(res):
|
||||
trainset = []
|
||||
testset = []
|
||||
@ -99,7 +100,9 @@ def split_train(res):
|
||||
|
||||
def calculate_checksums(text):
|
||||
"""
|
||||
@return fingerprints of a paragraphs in text. Paragraphs are separated by a blank line
|
||||
Paragraph separation must be compatible with text extraction. Are paragraphs separated with a blank line or a white line?
|
||||
|
||||
@return fingerprints of a paragraphs in text. Paragraphs are separated by a new line.
|
||||
"""
|
||||
checksums = []
|
||||
sizes = []
|
||||
@ -153,11 +156,11 @@ def is_link_good(link):
|
||||
return llink
|
||||
|
||||
def get_link_doc(link:str,status="frontlink")->dict:
|
||||
r = courlan.check_url(link)
|
||||
assert r is not None
|
||||
link,host = r
|
||||
domain = courlan.extract_domain(link)
|
||||
return {"url":link,"host":host,"domain":domain,"status":status,"created_at":dat.utcnow()}
|
||||
parsed = urllib.parse.urlparse(courlan.normalize_url(link))
|
||||
url = urllib.parse.urlunparse(parsed)
|
||||
tokens = parsed.netloc.split(".")
|
||||
domain = tokens[-2] + "." + tokens[-1]
|
||||
return {"url":link,"host":parsed.netloc,"domain":domain,"status":status,"created_at":dat.utcnow()}
|
||||
|
||||
|
||||
def fetch_page(link:str)->(str,str):
|
||||
@ -600,6 +603,12 @@ def domain_summary(db,hostname):
|
||||
for item in res:
|
||||
print(item)
|
||||
|
||||
def dropdb():
|
||||
myclient = pymongo.MongoClient(CONNECTION)
|
||||
print("write name of database to drop")
|
||||
dbname = sys.stdin.readline().strip()
|
||||
myclient.drop_database(dbname)
|
||||
|
||||
|
||||
def createdb():
|
||||
myclient = pymongo.MongoClient(CONNECTION)
|
||||
@ -617,7 +626,7 @@ def createdb():
|
||||
htmlcol.create_index("html_md5",unique=True)
|
||||
domaincol = db["domains"]
|
||||
domaincol.create_index("host",unique=True)
|
||||
domaincol.create_index(("average_fetch_characters",pymongo.DESCENDING))
|
||||
domaincol.create_index([("average_fetch_characters",pymongo.DESCENDING)])
|
||||
batchcol = db["batches"]
|
||||
batchcol.create_index("host")
|
||||
batchcol.create_index("created_at")
|
||||
@ -745,9 +754,11 @@ def import_html():
|
||||
for l in sys.stdin:
|
||||
hdoc = json.loads(l)
|
||||
url = hdoc["url"]
|
||||
# beautifusoup is to unify encoding
|
||||
html = BeautifulSoup(binascii.a2b_qp(hdoc["quoted_html"])).prettify()
|
||||
doc = extract_page(url,html)
|
||||
if doc is not None:
|
||||
print("------=====-")
|
||||
print(doc)
|
||||
status = index_page(db,url,url,html,doc)
|
||||
print(status)
|
||||
|
Loading…
Reference in New Issue
Block a user