misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implement parsing of robots.txt

Browse Source
This commit is contained in:
Simon Weald
2018-09-05 18:56:20 +01:00
parent f2c294ebdb
commit a3ec9451e3
3 changed files with 41 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
crawler.py
View File

@@ -4,16 +4,16 @@ Need a docstring.
'''
import argparse
from utils.helpers import (UrlPool, WebPage, sanitise_url)
from utils.helpers import (UrlPool, WebPage, RobotsTxt, sanitise_url)
from pprint import pprint
def init_crawler(base_url=None):
def init_crawler(base_url=None, robots=None):
'''
needs a docstring
'''
uncrawled_urls, crawled_urls = UrlPool(), UrlPool()
initial_page = WebPage(url=base_url, base_url=base_url)
initial_page = WebPage(url=base_url, base_url=base_url, robots=robots)
try:
initial_page.run()
@@ -45,7 +45,7 @@ def init_crawler(base_url=None):
return(uncrawled_urls, crawled_urls)
def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None):
def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=None):
'''
Needs a docstring
'''
@@ -53,7 +53,7 @@ def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None):
# pop url from pool
new_url = uncrawled_urls.remove_from_pool()
# create a WebPage object for the URL
current_page = WebPage(url=new_url, base_url=base_url)
current_page = WebPage(url=new_url, base_url=base_url, robots=robots)
try:
current_page.run()
_urls = current_page.list_urls()
@@ -72,9 +72,10 @@ def run(args=None):
needs a docstring.
'''
base_url = sanitise_url(args.url, base_url=True)
robots = RobotsTxt(base_url=base_url)
uncrawled_urls, crawled_urls = init_crawler(base_url)
process_pool(base_url, uncrawled_urls, crawled_urls)
uncrawled_urls, crawled_urls = init_crawler(base_url, robots)
process_pool(base_url, uncrawled_urls, crawled_urls, robots)
pprint(crawled_urls.pool)
print('{0} URLs crawled'.format(len(crawled_urls.pool)))