display count of crawled/uncrawled URLs whilst running

crawler.py

@@ -11,7 +11,7 @@ from utils.helpers import (UrlPool, WebPage, RobotsTxt, sanitise_url)
 
 def init_crawler(base_url=None, robots=None):
     '''
-    needs a docstring
+    Initialises the crawler by running the initial URL.
     '''
     uncrawled_urls, crawled_urls = UrlPool(), UrlPool()
     initial_page = WebPage(url=base_url, base_url=base_url, robots=robots)
@@ -48,7 +48,7 @@ def init_crawler(base_url=None, robots=None):
 
 def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=None):
     '''
-    Needs a docstring
+    Iterates over the pool of URLs and adds any discovered URLs.
     '''
     while uncrawled_urls.pool:
         # pop url from pool
@@ -67,6 +67,9 @@ def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=N
             if sanitised_url not in crawled_urls.pool:
                 uncrawled_urls.add_to_pool(url)
 
+        print('{0} URLs crawled, {1} remaining'.format(len(crawled_urls.pool),
+                len(uncrawled_urls.pool)))
+
 
 def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
     '''