added a docstring

2018-09-11 13:42:02 +01:00
parent c40c5cea50
commit 1af26f50f2
1 changed files with 3 additions and 0 deletions
--- a/utils/helpers.py
+++ b/utils/helpers.py
@@ -140,6 +140,9 @@ class RobotsTxt(object):
    '''

    def __init__(self, base_url=None):
+        '''
+        Manually retrieve robots.txt to allow us to set the user-agent.
+        '''
        self.base_url = base_url
        self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}