45 lines
1.1 KiB
Python
45 lines
1.1 KiB
Python
#!/usr/bin/env python
|
|
|
|
import re
|
|
import requests
|
|
|
|
|
|
def standardise_base_url(url):
|
|
'''
|
|
Standardise the URL to be scraped to ensure it
|
|
is added to relative URLs in a consistent manner.
|
|
'''
|
|
match_protocol = r'http(s?)\:\/\/'
|
|
|
|
if re.match(match_protocol, url):
|
|
base_url = url
|
|
else:
|
|
http_url = 'http://{0}'.format(url)
|
|
https_url = 'https://{0}'.format(url)
|
|
# attempt to discover which protocol is being used.
|
|
try:
|
|
result = requests.get(http_url)
|
|
if result.url.startswith('http'):
|
|
base_url = http_url
|
|
if result.url.startswith('https'):
|
|
base_url = https_url
|
|
except requests.exceptions.RequestException as e:
|
|
base_url = https_url
|
|
|
|
if base_url.endswith('/'):
|
|
base_url = base_url[:-1]
|
|
|
|
return base_url
|
|
|
|
|
|
def get_url_validation(base_url=None, url=None):
|
|
'''
|
|
Checks if a URL is valid. Can be absolute or relative.
|
|
'''
|
|
|
|
if url.startswith('/'):
|
|
full_url = '{0}{1}'.format(base_url, url)
|
|
if url.startswith(ffbase_url):
|
|
full_url = url
|
|
elif url.startswith('/'):
|