simplified url qualifier

utils/helpers.py

@@ -3,6 +3,8 @@
 Utilities to provide various misc functions.
 '''
 
+from urllib.parse import urljoin
+
 class UrlPoolManager(object):
     '''
     Object to manage the lifecycle of a pool of URLs.
@@ -41,21 +43,16 @@ def clean_base_url(url):
         # to HTTPS via a 301 redirect.
         base_url = "".join([protocol, url])
 
-    # strip the trailing slash to allow us to append
-    # relative URLs.
-    if base_url.endswith('/'):
-        base_url = base_url[:-1]
-
     return base_url
 
 
-# def get_url_validation(base_url=None, url=None):
+def get_url_validation(base_url=None, url=None):
-#     '''
+    '''
-#     Checks if a URL is valid. Can be absolute or relative.
+    Ensure any URLs discovered are absolute. If relative,
-#     '''
+    they will be appended to the base URL.
+    '''
 
-#     if url.startswith('/'):
+    if url.startswith('/'):
-#         full_url = '{0}{1}'.format(base_url, url)
+        return urljoin(base_url, url)
-#     if url.startswith(ffbase_url):
+    if url.startswith(base_url):
-#         full_url = url
+        return url
-#     elif url.startswith('/'):