misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add test for missing robots.txt

Browse Source
This commit is contained in:
Simon Weald
2018-09-18 10:53:13 +01:00
parent 51f988e1bc
commit 9a4e9ddfc7
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
test_helpers.py
View File

@@ -6,10 +6,12 @@ from utils.helpers import (RobotsTxt, standardise_url)
class TestRobots(unittest.TestCase): class TestRobots(unittest.TestCase):
base_url = 'http://eu.httpbin.org' base_url = 'http://eu.httpbin.org'
no_robots = 'https://www.simonweald.com'
test_paths = (('/', True), ('/deny', False)) test_paths = (('/', True), ('/deny', False))
robots = RobotsTxt(base_url=base_url) robots = RobotsTxt(base_url=base_url)
norobots = RobotsTxt(base_url=no_robots)
def test_robots_txt_deny(self): def test_robots_txt_deny(self):
''' '''
@@ -19,6 +21,13 @@ class TestRobots(unittest.TestCase):
result = self.robots.check(url=path) result = self.robots.check(url=path)
self.assertIs(result, allowed) self.assertIs(result, allowed)
def test_no_robots_txt(self):
'''
Ensure we can crawl if robots.txt isn't present.
'''
result = self.norobots.check(url='/')
self.assertTrue(result)
class TestUrls(unittest.TestCase): class TestUrls(unittest.TestCase):