diff --git a/mongo/mongocrawler.py b/mongo/mongocrawler.py index 92c0966..61db0f8 100644 --- a/mongo/mongocrawler.py +++ b/mongo/mongocrawler.py @@ -720,7 +720,6 @@ def visit(hostname,filter_content=True): doc = extract_page(final_link,html) extracted_pages.append((original_link,html,doc)) -<<<<<<< HEAD extracted_links = extract_links(links,responses,hostname,rules,"frontlink") index_links(db,extracted_links)