add test files

README.md

@@ -13,7 +13,7 @@ pip install -r requirements.txt
 Run:
 
 ```bash
-python crawler.py -u https://urltocrawl.com [-c 100]
+python async_crawler.py -u https://urltocrawl.com [-c 100]
 ```
 
 Flags:

async_crawler.py

@@ -8,7 +8,7 @@ URLs discovered will be crawled.
 
 The crawler takes a total of two arguments (concurrency is optional):
 
-    url: the base URL to begin the crawl from.
+    url: the root URL to begin the crawl from.
     concurrency: the maximum number of pages which may be crawled concurrently.
 '''
 
@@ -24,23 +24,36 @@ from utils.helpers import RobotsTxt, AsyncCrawler, standardise_url
 def sanity_checks(url=None):
     '''
     Runs some basic sanity checks before the crawler is initialised.
+
+    Accepts:
+      url: the root URL to be crawled.
+
+    Returns:
+      rooturl: a string containing avalidated and cleaned version of the
+               initial URL.
+      robots: an object which allows us to query whether a site may be crawled.
     '''
     # ensure we have a sensible URL to work with
-    baseurl = standardise_url(url=url)
+    rooturl = standardise_url(url=url)
     # get robots.txt
-    robots = RobotsTxt(base_url=baseurl)
+    robots = RobotsTxt(rooturl=rooturl)
 
     # fail early if robots denies all crawling
-    if not robots.check(url=baseurl):
+    if not robots.check(url=rooturl):
-        sys.exit("{baseurl} cannot be crawled (denied by robots.txt)".format(
+        sys.exit("{0} cannot be crawled (denied by robots.txt)".format(
-            baseurl=baseurl))
+                 rooturl))
 
-    return(baseurl, robots)
+    return(rooturl, robots)
 
 
-def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
+def render_sitemap(rooturl=None, crawled_urls=None, runtime=None):
     '''
     Renders the sitemap to an HTML file.
+
+    Accepts:
+        rooturl: string containing the root URL
+        crawled_urls: set containing discovered URLs
+        runtime: int representing run time of AsyncCrawler
     '''
     urlcount = len(crawled_urls)
     sorted_urls = sorted(crawled_urls)
@@ -49,7 +62,7 @@ def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
         loader=jinja2.FileSystemLoader('templates')
     ).get_template('sitemap.html.j2')
 
-    rendered_html = template.render(base_url=base_url, urlcount=urlcount,
+    rendered_html = template.render(rooturl=rooturl, urlcount=urlcount,
                                     urls=sorted_urls, runtime=runtime)
 
     with open('sitemap.html', 'w') as outfile:
@@ -65,10 +78,10 @@ def main():
     '''
     starttime = datetime.now()
 
-    baseurl, robots = sanity_checks(url=args.url)
+    rooturl, robots = sanity_checks(url=args.url)
 
     # create a crawler
-    async_crawler = AsyncCrawler(baseurl=baseurl, robots=robots,
+    async_crawler = AsyncCrawler(rooturl=rooturl, robots=robots,
                                  concurrency=args.concurrency)
 
     # create a task to run the crawler, run the loop and then gather the
@@ -81,7 +94,7 @@ def main():
 
     runtime = int((datetime.now() - starttime).total_seconds())
 
-    render_sitemap(base_url=baseurl, crawled_urls=results, runtime=runtime)
+    render_sitemap(rooturl=rooturl, crawled_urls=results, runtime=runtime)
 
 
 if __name__ == '__main__':

notes.md

@@ -18,39 +18,9 @@
   * better exception handling
   * randomise output filename
 
-### Async bits
+### talking points
 
-in `__main__`:
+ - token bucket algo to enforce n requests per second
-
+   - read up on bucket algo types
-```python
+ - re-structuring AsyncCrawler to be more testable
-loop = asyncio.get_event_loop()
+ - use exponential backoff algo?
-try:
-    loop.run_until_complete(main())
-finally:
-    loop.close()
-```
-
-  * initialises loop and runs it to completion
-  * needs to handle errors (try/except/finally)
-
-```python
-async def run(args=None):
-    tasks = []
-
-    for url in pool:
-        tasks.append(url)
-    # for i in range(10):
-        # tasks.append(asyncio.ensure_future(myCoroutine(i)))
-
-    # gather completed tasks
-    await asyncio.gather(*tasks)
-```
-
-Getting the contents of the page needs to be async too
-
-```python
-async def get_source():
-    blah
-    blah
-    await urlopen(url)
-```

templates/sitemap.html.j2

@@ -4,7 +4,7 @@
 </head>
 <body>
 <p>
-Crawled {{ urlcount }} URLs on {{ base_url }} in ~{{ runtime }} seconds.
+Crawled {{ urlcount }} URLs on {{ rooturl }} in ~{{ runtime }} seconds.
 <ul>
 {% for url in urls %}
   <li><a href="{{ url }}">{{ url }}</a></li>

test/files/find_all_urls.html

@@ -0,0 +1,10 @@
+<html>
+<body>
+<p>
+<ul>
+  <li><a href="http://eu.httpbin.org/a/">http://eu.httpbin.org/a/</a></li>
+  <li><a href="http://eu.httpbin.org/b/">http://eu.httpbin.org/b/</a></li>
+  <li><a href="http://eu.httpbin.org/c/">http://eu.httpbin.org/c/</a></li>
+</ul>
+</body>
+</html>

test_helpers.py

@@ -1,34 +1,65 @@
 #!/usr/bin/env python
 
 import unittest
-from utils.helpers import (sanitise_url)
+from unittest import mock
+from utils.helpers import RobotsTxt, standardise_url
+
+
+class TestRobots(unittest.TestCase):
+
+    rooturl = 'http://eu.httpbin.org'
+    no_robots = 'https://www.simonweald.com'
+
+    test_paths = (('/', True), ('/deny', False))
+
+    robots = RobotsTxt(rooturl=rooturl)
+    norobots = RobotsTxt(rooturl=no_robots)
+
+    def test_robots_txt_deny(self):
+        '''
+        Asserts result is True or False.
+        '''
+        for path, allowed in self.test_paths:
+            result = self.robots.check(url=path)
+            self.assertIs(result, allowed)
+
+    def test_no_robots_txt(self):
+        '''
+        Ensure we can crawl if robots.txt isn't present.
+        '''
+        result = self.norobots.check(url='/')
+        self.assertTrue(result)
+
 
 class TestUrls(unittest.TestCase):
 
-    base_url_list = (('eu.httpbin.org', 'http://eu.httpbin.org'),
+    rooturl = 'http://eu.httpbin.org'
-                     ('www.simonweald.com', 'http://www.simonweald.com'),
-                     ('http://www.github.com/', 'http://www.github.com'),
-                     ('https://www.github.com', 'https://www.github.com'))
 
-    urls_to_clean = (('https://www.github.com/', 'https://www.github.com/'),
+    rooturl_list = (('eu.httpbin.org', 'http://eu.httpbin.org'),
-                     ('https://github.com/?foo=bar', 'https://github.com/'),
+                     ('http://eu.httpbin.org/', 'http://eu.httpbin.org'),
-                     ('https://github.com/#anchor', 'https://github.com/'))
+                     ('https://eu.httpbin.org', 'https://eu.httpbin.org'))
 
+    urls_to_clean = (('http://eu.httpbin.org', 'http://eu.httpbin.org'),
+                     ('http://eu.httpbin.org/some/path/', 'http://eu.httpbin.org/some/path/'),
+                     ('http://eu.httpbin.org/index.html','http://eu.httpbin.org/index.html'),
+                     ('http://eu.httpbin.org/index.html?foo=bar', 'http://eu.httpbin.org/index.html'),
+                     ('http://eu.httpbin.org/index.html#anchor', 'http://eu.httpbin.org/index.html'))
 
-    def test_sanitise_base_url(self):
+    def test_standardise_rooturl(self):
         '''
-        Tests whether a URL's protocol can be discovered if not provided.
+        Tests whether a base URL can be standardised to the format
+        proto://[sub].domain.tld.
         '''
-        for url, target in self.base_url_list:
+        for url, target in self.rooturl_list:
-            result = sanitise_url(url, base_url=True)
+            result = standardise_url(url)
             self.assertEqual(result, target)
 
-    def test_sanitise_url(self):
+    def test_standardise_url(self):
         '''
-        Tests whether a URL's protocol can be discovered if not provided.
+        Ensure that fragments/anchors etc are stripped.
         '''
         for url, target in self.urls_to_clean:
-            result = sanitise_url(url)
+            result = standardise_url(url, rooturl=self.rooturl)
             self.assertEqual(result, target)
 
 

utils/helpers.py

@@ -14,11 +14,22 @@ import urllib.robotparser
 
 class AsyncCrawler(object):
     '''
-    docstring
+    A concurrent recursive web crawler.
+
+    A recursive web crawler which finds all URLs local to the domains specified
+    in the `rooturl` argument.
+
+    Arguments:
+        rooturl:        Root domain to begin crawling.
+        robots:         RobotsTxt object for the rooturl.
+        concurrency:    number of concurrent pages to crawl.
+
+    Returns:
+        All discovered pages in a set.
     '''
 
-    def __init__(self, baseurl=None, robots=None, concurrency=None):
+    def __init__(self, rooturl=None, robots=None, concurrency=None):
-        self.baseurl = baseurl
+        self.rooturl = rooturl
         self.robots = robots
         self.crawled = set()
         self.headers = {'Accept-Encoding': 'gzip, deflate',
@@ -28,7 +39,7 @@ class AsyncCrawler(object):
 
     async def crawl_url(self, url=None):
         '''
-        Crawls the given URL and finds all new URLs in the initial page.
+        Crawls the given URL and finds all new URLs in the given page.
         '''
         urls = []
         source = await self.get_source(url)
@@ -43,7 +54,7 @@ class AsyncCrawler(object):
         allows it.
         '''
         # ensure the URL is in a sane format
-        url = standardise_url(url=url, base_url=self.baseurl)
+        url = standardise_url(url=url, rooturl=self.rooturl)
 
         if url and self.robots.check(url=url):
             return url
@@ -53,14 +64,15 @@ class AsyncCrawler(object):
     async def get_source(self, url=None):
         '''
         Obtains the URL's source, provided it is HTML. Usage of semaphores
-        ensures only a certain number of coroutines can run at once.
+        ensures only a certain number of coroutines can run at any given
+        time.
         '''
         async with self.semaphore:
             async with self.client_session.head(url, timeout=5) as head:
                 try:
-                    data = await head.read()
+                    _ = await head.read()
-                except Exception as e:
+                except Exception:
-                    print(e)
+                    pass
             if 'text/html' in head.headers['Content-Type']:
                 async with self.client_session.get(url, timeout=5) as resp:
                     try:
@@ -68,6 +80,8 @@ class AsyncCrawler(object):
                         return source
                     except Exception:
                         return None
+            else:
+                return None
 
     def find_all_urls(self, source=None):
         '''
@@ -99,21 +113,22 @@ class AsyncCrawler(object):
         for url in urls:
             # mark the URL as seen.
             self.crawled.add(url)
-            # create an coroutine to crawl the URL.
+            # create a task to crawl the URL.
             tasks.append(self.crawl_url(url))
 
         # wait for all tasks to complete.
         for task in asyncio.as_completed(tasks):
             urls = None
             try:
+                # try getting all tasks as completed.
                 urls = await task
-            except Exception as e:
+            except Exception:
-                print(e)
+                # skip until all tasks have completed.
+                pass
 
             # add the URLs to a set to be returned.
             if urls:
                 for url in urls:
-                    print('Found: {0}'.format(url))
                     all_urls.add(url)
 
         return all_urls
@@ -125,15 +140,17 @@ class AsyncCrawler(object):
         '''
         self.client_session = aiohttp.ClientSession(headers=self.headers)
         to_crawl = []
-        to_crawl.append(self.baseurl)
+        # add the root URL to initialise the crawler.
+        to_crawl.append(self.rooturl)
 
-        print('Crawling: {0}'.format(self.baseurl))
+        print('Crawling: {0}'.format(self.rooturl))
         while len(to_crawl) > 0:
             discovered_urls = await self.run(urls=to_crawl)
-            # empty toe crawl list and then add all newly discovered URLs for
+            # empty to_crawl list and then add all newly discovered URLs for
             # the next iteration.
             to_crawl.clear()
             to_crawl.extend(discovered_urls)
+            print('{0} URLs crawled'.format(len(self.crawled)))
 
         # close the ssions once all URLs have been crawled.
         await self.client_session.close()
@@ -143,19 +160,23 @@ class AsyncCrawler(object):
 
 class RobotsTxt(object):
     '''
-    needs a docstring
+    Retrieve and query robots.txt for a given domain.
+
+    Retrieves and parses robots.txt for the given domain. Calling the check()
+    method returns True or False depending on whether crawling of that given
+    URL is allowed.
     '''
 
-    def __init__(self, base_url=None):
+    def __init__(self, rooturl=None):
         '''
         Manually retrieve robots.txt to allow us to set the user-agent (works
         around sites which disallow access to robots.txt without a sane
         user-agent).
         '''
-        self.base_url = base_url
+        self.rooturl = rooturl
         self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
 
-        robots_url = urljoin(self.base_url, 'robots.txt')
+        robots_url = urljoin(self.rooturl, 'robots.txt')
         request = urllib.request.Request(robots_url, headers=self.headers)
 
         robots = urllib.robotparser.RobotFileParser()
@@ -180,16 +201,16 @@ class RobotsTxt(object):
         return self.robots.can_fetch("*", url)
 
 
-def standardise_url(url=None, base_url=None):
+def standardise_url(url=None, rooturl=None):
     '''
-    If `base_url` is None then we attempt to standarise the URL to ensure it
+    If `rooturl` is None then we attempt to standarise the URL to ensure it
     can be prepended to relative URLs. If no scheme has been provided then we
     default to http as any sane https-only site should 301 redirect http to
     https.
 
-    If `base_url` is set, we standardise URLs to strip queries and fragments
+    If `rooturl` is set, we standardise URLs to strip queries and fragments
     (we don't want to scrape in-page anchors etc). Any relative URLs will be
-    appended to the base url.
+    appended to the root url.
 
     Returns a standardised URL as a string.
     '''
@@ -200,7 +221,7 @@ def standardise_url(url=None, base_url=None):
 
     split_url = urlsplit(url)
 
-    if not base_url:
+    if not rooturl:
         # This will sanitise the initial url provided by the user.
         if split_url.scheme and split_url.scheme.startswith('http'):
             return "".join([split_url.scheme, delim, split_url.netloc])
@@ -209,10 +230,10 @@ def standardise_url(url=None, base_url=None):
     else:
         # if url.endswith(file_extensions):
         # Sanitise discovered URLs. We already expect them in the format
-        # protocol://base_url/path
+        # protocol://rooturl/path
         if url.startswith('/'):
-            return urljoin(base_url, split_url.path)
+            return urljoin(rooturl, split_url.path)
-        elif url.startswith(base_url):
+        elif url.startswith(rooturl):
             return "".join([split_url.scheme, delim, split_url.netloc,
                             split_url.path])