misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

display count of crawled/uncrawled URLs whilst running

Browse Source
This commit is contained in:
Simon Weald
2018-09-09 22:35:55 +01:00
parent 9e754a5584
commit a523154848
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crawler.py
View File

@@ -11,7 +11,7 @@ from utils.helpers import (UrlPool, WebPage, RobotsTxt, sanitise_url)
def init_crawler(base_url=None, robots=None): def init_crawler(base_url=None, robots=None):
''' '''
needs a docstring Initialises the crawler by running the initial URL.
''' '''
uncrawled_urls, crawled_urls = UrlPool(), UrlPool() uncrawled_urls, crawled_urls = UrlPool(), UrlPool()
initial_page = WebPage(url=base_url, base_url=base_url, robots=robots) initial_page = WebPage(url=base_url, base_url=base_url, robots=robots)
@@ -48,7 +48,7 @@ def init_crawler(base_url=None, robots=None):
def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=None): def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=None):
''' '''
Needs a docstring Iterates over the pool of URLs and adds any discovered URLs.
''' '''
while uncrawled_urls.pool: while uncrawled_urls.pool:
# pop url from pool # pop url from pool
@@ -67,6 +67,9 @@ def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=N
if sanitised_url not in crawled_urls.pool: if sanitised_url not in crawled_urls.pool:
uncrawled_urls.add_to_pool(url) uncrawled_urls.add_to_pool(url)
print('{0} URLs crawled, {1} remaining'.format(len(crawled_urls.pool),
len(uncrawled_urls.pool)))
def render_sitemap(base_url=None, crawled_urls=None, runtime=None): def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
''' '''

2
notes.md
View File

@@ -13,6 +13,6 @@
* ~~investigate using [tldextract](https://github.com/john-kurkowski/tldextract) to match urls~~ #wontfix * ~~investigate using [tldextract](https://github.com/john-kurkowski/tldextract) to match urls~~ #wontfix
* ~~implement parsing of [robots.txt](http://docs.w3cub.com/python~3.6/library/urllib.robotparser/)~~ * ~~implement parsing of [robots.txt](http://docs.w3cub.com/python~3.6/library/urllib.robotparser/)~~
* ~~investigate [gzip encoding](https://stackoverflow.com/questions/36383227/avoid-downloading-images-using-beautifulsoup-and-urllib-request)~~ * ~~investigate [gzip encoding](https://stackoverflow.com/questions/36383227/avoid-downloading-images-using-beautifulsoup-and-urllib-request)~~
* implement some kind of progress display * ~~implement some kind of progress display~~
* async * async
* better exception handling * better exception handling