From 1af26f50f23c1d0b5a2c9344b4879277041b4432 Mon Sep 17 00:00:00 2001
From: Simon Weald <simon@simonweald.com>
Date: Tue, 11 Sep 2018 13:42:02 +0100
Subject: [PATCH] added a docstring

---
 utils/helpers.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/utils/helpers.py b/utils/helpers.py
index 6a43cba..3f9f1a3 100644
--- a/utils/helpers.py
+++ b/utils/helpers.py
@@ -140,6 +140,9 @@ class RobotsTxt(object):
     '''
 
     def __init__(self, base_url=None):
+        '''
+        Manually retrieve robots.txt to allow us to set the user-agent.
+        '''
         self.base_url = base_url
         self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}