more comments and progress output

async_crawler.py

@@ -51,7 +51,8 @@ def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
 
 def main():
     '''
-    docstring
+    Main function, responsible for prepping and running the crawler and
+    rendering the sitemap.
     '''
     starttime = datetime.now()
 
@@ -60,11 +61,13 @@ def main():
     # create a crawler
     async_crawler = AsyncCrawler(baseurl=baseurl, robots=robots, concurrency=args.concurrency)
 
+    # create a task to run the crawler, run the loop and then gather the results.
     task = asyncio.Task(async_crawler.main())
     loop = asyncio.get_event_loop()
     loop.run_until_complete(task)
     loop.close()
-    results = task.result()
+    results = sorted(task.result())
+
     runtime = int((datetime.now() - starttime).total_seconds())
 
     render_sitemap(base_url=baseurl, crawled_urls=results, runtime=runtime)