remove unecessary function
This commit is contained in:
@@ -4,7 +4,7 @@ Need a docstring.
|
||||
'''
|
||||
|
||||
import argparse
|
||||
from utils.helpers import (UrlPool, WebPage, sanitise_url, qualify_url)
|
||||
from utils.helpers import (UrlPool, WebPage, sanitise_url)
|
||||
|
||||
def init_crawler(base_url=None):
|
||||
'''
|
||||
|
||||
@@ -118,16 +118,3 @@ def sanitise_url(url):
|
||||
base_url = "".join([default_proto, delim, split_url.path])
|
||||
|
||||
return base_url
|
||||
|
||||
|
||||
def qualify_url(base_url=None, url=None):
|
||||
'''
|
||||
Ensure any URLs discovered are absolute. If relative,
|
||||
they will be appended to the base URL. Returns an
|
||||
absolute URL as a string.
|
||||
'''
|
||||
|
||||
if url.startswith('/'):
|
||||
return urljoin(base_url, url)
|
||||
if url.startswith(base_url):
|
||||
return url
|
||||
|
||||
Reference in New Issue
Block a user