misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove unecessary function

Browse Source
This commit is contained in:
Simon Weald
2018-08-31 19:16:08 +01:00
parent 03554fde80
commit c436016e0c
2 changed files with 1 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler.py
View File

@@ -4,7 +4,7 @@ Need a docstring.
''' '''
import argparse import argparse
from utils.helpers import (UrlPool, WebPage, sanitise_url, qualify_url) from utils.helpers import (UrlPool, WebPage, sanitise_url)
def init_crawler(base_url=None): def init_crawler(base_url=None):
''' '''

13
utils/helpers.py
View File

@@ -118,16 +118,3 @@ def sanitise_url(url):
base_url = "".join([default_proto, delim, split_url.path]) base_url = "".join([default_proto, delim, split_url.path])
return base_url return base_url
def qualify_url(base_url=None, url=None):
'''
Ensure any URLs discovered are absolute. If relative,
they will be appended to the base URL. Returns an
absolute URL as a string.
'''
if url.startswith('/'):
return urljoin(base_url, url)
if url.startswith(base_url):
return url