simplify UrlPoolManager to use a set instead of a dict

utils/helpers.py

@@ -9,28 +9,22 @@ class UrlPoolManager(object):
     '''
 
     def __init__(self):
-        self.url_pool = dict()
+        self.url_pool = set()
-        self.not_crawled = 0
-        self.crawled = 1
-        self.invalid = 2
 
     def check_duplicate(self, new_url):
-        for url, status in self.url_pool.items():
+        '''
-            if url == new_url:
+        Checks if a URL exists in the current pool.
-                return True
+        '''
-            else:
+        if new_url in self.url_pool:
-                return False
+            return True
+        else:
+            return False
 
     def invalidate_url(self, url):
-        self.url_pool[url] = self.invalid
+        self.url_pool.remove(url)
 
     def add_to_list(self, url):
-        self.url_pool[url] = self.not_crawled
+        self.url_pool.add(url)
-        # calculate depth
-        # add link, crawled status to url_pool
-
-    def mark_as_crawled(self, url):
-        self.url_pool[url] = self.crawled
 
 
 def clean_base_url(url):