remove unecessary function

crawler.py

@@ -4,7 +4,7 @@ Need a docstring.
 '''
 
 import argparse
-from utils.helpers import (UrlPool, WebPage, sanitise_url, qualify_url)
+from utils.helpers import (UrlPool, WebPage, sanitise_url)
 
 def init_crawler(base_url=None):
     '''

utils/helpers.py

@@ -41,15 +41,24 @@ class WebPage(object):
 
 
     def get_source(self):
+        '''
+        Retrieve a page's source.
+        '''
+
         request = urllib.request.Request(self.url, headers=self.headers)
         page = urllib.request.urlopen(request)
         self.source = page.read()
 
 
     def find_links(self):
+        '''
+        Find all URLs on a page and ensure they are absolute. If they are
+        relative then they will be appended to the base URL.
+        '''
+        hrefs = set()
+
         soup = BeautifulSoup(self.source, 'html.parser')
         links = soup.find_all('a')
-        hrefs = set()
 
         for link in links:
             if link['href'].startswith('/'):
@@ -61,6 +70,11 @@ class WebPage(object):
 
 
     def parse_urls(self):
+        '''
+        Iterate through the list of discovered URLs and add them to the
+        pool if they start with the base URL.
+        '''
+
         self.urls_to_crawl = set()
         for url in self.discovered_hrefs:
             if url.startswith(self.url):
@@ -104,16 +118,3 @@ def sanitise_url(url):
         base_url = "".join([default_proto, delim, split_url.path])
 
     return base_url
-
-
-def qualify_url(base_url=None, url=None):
-    '''
-    Ensure any URLs discovered are absolute. If relative,
-    they will be appended to the base URL. Returns an
-    absolute URL as a string.
-    '''
-
-    if url.startswith('/'):
-        return urljoin(base_url, url)
-    if url.startswith(base_url):
-        return url