62 lines
1.4 KiB
Python
62 lines
1.4 KiB
Python
#!/usr/bin/env python
|
|
'''
|
|
Utilities to provide various misc functions.
|
|
'''
|
|
|
|
class UrlPoolManager(object):
|
|
'''
|
|
Object to manage the lifecycle of a pool of URLs.
|
|
'''
|
|
|
|
def __init__(self):
|
|
self.url_pool = set()
|
|
|
|
def check_duplicate(self, new_url):
|
|
'''
|
|
Checks if a URL exists in the current pool.
|
|
'''
|
|
if new_url in self.url_pool:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def invalidate_url(self, url):
|
|
self.url_pool.remove(url)
|
|
|
|
def add_to_list(self, url):
|
|
self.url_pool.add(url)
|
|
|
|
|
|
def clean_base_url(url):
|
|
'''
|
|
Standardise the URL to be scraped to ensure it
|
|
is added to relative URLs in a consistent manner.
|
|
'''
|
|
protocol = 'http://'
|
|
|
|
if url.startswith('http'):
|
|
base_url = url
|
|
else:
|
|
# otherwise assume HTTP as any sane site should upgrade
|
|
# to HTTPS via a 301 redirect.
|
|
base_url = "".join([protocol, url])
|
|
|
|
# strip the trailing slash to allow us to append
|
|
# relative URLs.
|
|
if base_url.endswith('/'):
|
|
base_url = base_url[:-1]
|
|
|
|
return base_url
|
|
|
|
|
|
# def get_url_validation(base_url=None, url=None):
|
|
# '''
|
|
# Checks if a URL is valid. Can be absolute or relative.
|
|
# '''
|
|
|
|
# if url.startswith('/'):
|
|
# full_url = '{0}{1}'.format(base_url, url)
|
|
# if url.startswith(ffbase_url):
|
|
# full_url = url
|
|
# elif url.startswith('/'):
|