diff --git a/crawler.py b/crawler.py index ef01771..abe3d93 100644 --- a/crawler.py +++ b/crawler.py @@ -4,7 +4,7 @@ Need a docstring. ''' import argparse -from utils.helpers import (UrlPool, WebPage, sanitise_url, qualify_url) +from utils.helpers import (UrlPool, WebPage, sanitise_url) def init_crawler(base_url=None): ''' diff --git a/utils/helpers.py b/utils/helpers.py index 4c15603..83a9a74 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -118,16 +118,3 @@ def sanitise_url(url): base_url = "".join([default_proto, delim, split_url.path]) return base_url - - -def qualify_url(base_url=None, url=None): - ''' - Ensure any URLs discovered are absolute. If relative, - they will be appended to the base URL. Returns an - absolute URL as a string. - ''' - - if url.startswith('/'): - return urljoin(base_url, url) - if url.startswith(base_url): - return url