dano/websucker-pip
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zz

Browse Source
This commit is contained in:
Daniel Hládek 2023-03-12 06:56:08 +01:00
parent e728134bf6
commit 3993a61899
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mongo/mongocwarler.py
View File

@ -263,8 +263,13 @@ def create_indices(db):
htmlcol = db["html"] htmlcol = db["html"]
htmlcol.create_index({"url":1}) htmlcol.create_index({"url":1})
@click.group()
def cli():
pass
@click.command() @click.command()
def simple_visit(start_link=None): @click.argument("start_link")
def simple_visit(start_link):
start_link,domain = courlan.check_url(start_link) start_link,domain = courlan.check_url(start_link)
myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/") myclient = pymongo.MongoClient("mongodb://root:example@localhost:27017/")
db=myclient["crawler"] db=myclient["crawler"]
@ -282,3 +287,6 @@ def simple_visit(start_link=None):
front_links = get_links(db,domain,"backlink",bl) front_links = get_links(db,domain,"backlink",bl)
process_links(db,domain,"backlink",links,rules=rules) process_links(db,domain,"backlink",links,rules=rules)
link_summary(db,domain) link_summary(db,domain)
if __name__ == "__main__":
cli()