diff --git a/utils/helpers.py b/utils/helpers.py
index dc522fa..8133cb5 100644
--- a/utils/helpers.py
+++ b/utils/helpers.py
@@ -138,17 +138,17 @@ class RobotsTxt(object):
         robots_url = urljoin(self.base_url, 'robots.txt')
         request = urllib.request.Request(robots_url, headers=self.headers)
 
+        robots = urllib.robotparser.RobotFileParser()
+        robots.set_url(robots_url)
+
         try:
             response = urllib.request.urlopen(request, timeout=5)
-        except urllib.error.HTTPError as err:
-            print(err)
+        except urllib.error.HTTPError:
+            robots.allow_all = True
         else:
             data = response.read()
             decoded_data = data.decode("utf-8").splitlines()
-
-        robots = urllib.robotparser.RobotFileParser()
-        robots.set_url(robots_url)
-        robots.parse(decoded_data)
+            robots.parse(decoded_data)
 
         self.robots = robots