From a523154848b7231418f9069ab868f4d6a23f7df6 Mon Sep 17 00:00:00 2001 From: Simon Weald Date: Sun, 9 Sep 2018 22:35:55 +0100 Subject: [PATCH] display count of crawled/uncrawled URLs whilst running --- crawler.py | 7 +++++-- notes.md | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crawler.py b/crawler.py index 3c6bf61..5744248 100644 --- a/crawler.py +++ b/crawler.py @@ -11,7 +11,7 @@ from utils.helpers import (UrlPool, WebPage, RobotsTxt, sanitise_url) def init_crawler(base_url=None, robots=None): ''' - needs a docstring + Initialises the crawler by running the initial URL. ''' uncrawled_urls, crawled_urls = UrlPool(), UrlPool() initial_page = WebPage(url=base_url, base_url=base_url, robots=robots) @@ -48,7 +48,7 @@ def init_crawler(base_url=None, robots=None): def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=None): ''' - Needs a docstring + Iterates over the pool of URLs and adds any discovered URLs. ''' while uncrawled_urls.pool: # pop url from pool @@ -67,6 +67,9 @@ def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=N if sanitised_url not in crawled_urls.pool: uncrawled_urls.add_to_pool(url) + print('{0} URLs crawled, {1} remaining'.format(len(crawled_urls.pool), + len(uncrawled_urls.pool))) + def render_sitemap(base_url=None, crawled_urls=None, runtime=None): ''' diff --git a/notes.md b/notes.md index d40faad..8ef5f2d 100644 --- a/notes.md +++ b/notes.md @@ -13,6 +13,6 @@ * ~~investigate using [tldextract](https://github.com/john-kurkowski/tldextract) to match urls~~ #wontfix * ~~implement parsing of [robots.txt](http://docs.w3cub.com/python~3.6/library/urllib.robotparser/)~~ * ~~investigate [gzip encoding](https://stackoverflow.com/questions/36383227/avoid-downloading-images-using-beautifulsoup-and-urllib-request)~~ - * implement some kind of progress display + * ~~implement some kind of progress display~~ * async * better exception handling