From 0517e5bc56ed09e657ab8905b623e4c426c8d793 Mon Sep 17 00:00:00 2001
From: Simon Weald <simon@simonweald.com>
Date: Fri, 31 Aug 2018 19:02:21 +0100
Subject: [PATCH] crawler now initialises and populates crawled pool with urls
 it finds

---
 crawler.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/crawler.py b/crawler.py
index 8d3839c..ef01771 100644
--- a/crawler.py
+++ b/crawler.py
@@ -10,6 +10,21 @@ def init_crawler(base_url=None):
     '''
     needs a docstring
     '''
+    uncrawled_urls, crawled_urls = UrlPool(), UrlPool()
+    initial_page = WebPage(base_url)
+
+    try:
+        initial_urls = initial_page.run()
+    except Exception as e:
+        print(e)
+
+    for url in initial_urls:
+        try:
+            uncrawled_urls.add_to_pool(url)
+        except Exception as e:
+            print(e)
+
+    print(uncrawled_urls.url_pool)
 
 
 def run(args=None):
@@ -17,7 +32,9 @@ def run(args=None):
     needs a docstring.
     '''
     base_url = sanitise_url(args.url)
-    print(base_url)
+
+    init_crawler(base_url)
+
 
 if __name__ == '__main__':