diff --git a/crawler.py b/crawler.py index 5744248..5f349d2 100644 --- a/crawler.py +++ b/crawler.py @@ -5,6 +5,7 @@ Need a docstring. import argparse import jinja2 +import os from datetime import datetime from utils.helpers import (UrlPool, WebPage, RobotsTxt, sanitise_url) @@ -87,7 +88,7 @@ def render_sitemap(base_url=None, crawled_urls=None, runtime=None): with open('sitemap.html', 'w') as outfile: outfile.write(rendered_html) - print('Sitemap available at sitemap.html') + print('Sitemap available at {0}/sitemap.html'.format(os.getcwd())) def run(args=None): @@ -106,9 +107,6 @@ def run(args=None): render_sitemap(base_url=base_url, crawled_urls=crawled_urls.pool, runtime=runtime) - # pprint(crawled_urls.pool) - # print('{0} URLs crawled'.format(len(crawled_urls.pool))) - if __name__ == '__main__':