misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added a docstring

Browse Source
This commit is contained in:
Simon Weald
2018-09-11 13:42:02 +01:00
parent c40c5cea50
commit 1af26f50f2
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
utils/helpers.py
View File

@@ -140,6 +140,9 @@ class RobotsTxt(object):
''' '''
def __init__(self, base_url=None): def __init__(self, base_url=None):
'''
Manually retrieve robots.txt to allow us to set the user-agent.
'''
self.base_url = base_url self.base_url = base_url
self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'} self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}