update docs

README.md

@@ -2,7 +2,7 @@
 
 ## Requirements
 
-This crawler was written in 3.7.0 to take advantage of the latest `asyncio` features.
+This crawler requires at least Python 3.5 in order to utilise the async/await keywords from `asyncio`.
 
 Install required modules:
 
@@ -13,9 +13,16 @@ pip install -r requirements.txt
 Run:
 
 ```bash
-python crawler.py -u https://urltocrawl.com
+python async_crawler.py -u https://urltocrawl.com [-c 100]
 ```
 
+Flags:
+
+  - `-u/--url https://url.com`
+    - The base URL is required.
+  - `-c/--concurrency 100`
+    - Specifying concurrency value is optional (defaults to 100).
+
 ## Results
 
-The resulting sitemap will be output in the root of this directory as `sitemap.html`
+The resulting sitemap will be output to the root of this directory as `sitemap.html`

async_crawler.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+'''
+Asynchronous web crawler written in Python 3.5+.
+
+This script will respect the site's `robots.txt`, if one exists. If not, all
+URLs discovered will be crawled.
+
+The crawler takes a total of two arguments (concurrency is optional):
+
+    url: the root URL to begin the crawl from.
+    concurrency: the maximum number of pages which may be crawled concurrently.
+'''
+
+import argparse
+import asyncio
+from datetime import datetime
+import jinja2
+import os
+import sys
+from utils.helpers import RobotsTxt, AsyncCrawler, standardise_url
+
+
+def sanity_checks(url=None):
+    '''
+    Runs some basic sanity checks before the crawler is initialised.
+
+    Accepts:
+      url: the root URL to be crawled.
+
+    Returns:
+      rooturl: a string containing avalidated and cleaned version of the
+               initial URL.
+      robots: an object which allows us to query whether a site may be crawled.
+    '''
+    # ensure we have a sensible URL to work with
+    rooturl = standardise_url(url=url)
+    # get robots.txt
+    robots = RobotsTxt(rooturl=rooturl)
+
+    # fail early if robots denies all crawling
+    if not robots.check(url=rooturl):
+        sys.exit("{0} cannot be crawled (denied by robots.txt)".format(
+                 rooturl))
+
+    return(rooturl, robots)
+
+
+def render_sitemap(rooturl=None, crawled_urls=None, runtime=None):
+    '''
+    Renders the sitemap to an HTML file.
+
+    Accepts:
+        rooturl: string containing the root URL
+        crawled_urls: set containing discovered URLs
+        runtime: int representing run time of AsyncCrawler
+    '''
+    urlcount = len(crawled_urls)
+    sorted_urls = sorted(crawled_urls)
+
+    template = jinja2.Environment(
+        loader=jinja2.FileSystemLoader('templates')
+    ).get_template('sitemap.html.j2')
+
+    rendered_html = template.render(rooturl=rooturl, urlcount=urlcount,
+                                    urls=sorted_urls, runtime=runtime)
+
+    with open('sitemap.html', 'w') as outfile:
+        outfile.write(rendered_html)
+
+    print('Sitemap available at {0}/sitemap.html'.format(os.getcwd()))
+
+
+def main():
+    '''
+    Main function, responsible for prepping and running the crawler and
+    rendering the sitemap.
+    '''
+    starttime = datetime.now()
+
+    rooturl, robots = sanity_checks(url=args.url)
+
+    # create a crawler
+    async_crawler = AsyncCrawler(rooturl=rooturl, robots=robots,
+                                 concurrency=args.concurrency)
+
+    # create a task to run the crawler, run the loop and then gather the
+    # results.
+    task = asyncio.Task(async_crawler.main())
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(task)
+    loop.close()
+    results = sorted(task.result())
+
+    runtime = int((datetime.now() - starttime).total_seconds())
+
+    render_sitemap(rooturl=rooturl, crawled_urls=results, runtime=runtime)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Recursive web crawler')
+    parser.add_argument("-u", "--url", required=True, help="Initial url")
+    parser.add_argument("-c", "--concurrency", required=False, type=int,
+                        default=100, help="Max pages to crawl concurrently")
+    args = parser.parse_args()
+
+    main()

notes.md

@@ -18,39 +18,9 @@
   * better exception handling
   * randomise output filename
 
-### Async bits
+### talking points
 
-in `__main__`:
-
-```python
-loop = asyncio.get_event_loop()
-try:
-    loop.run_until_complete(main())
-finally:
-    loop.close()
-```
-
-  * initialises loop and runs it to completion
-  * needs to handle errors (try/except/finally)
-
-```python
-async def run(args=None):
-    tasks = []
-
-    for url in pool:
-        tasks.append(url)
-    # for i in range(10):
-        # tasks.append(asyncio.ensure_future(myCoroutine(i)))
-
-    # gather completed tasks
-    await asyncio.gather(*tasks)
-```
-
-Getting the contents of the page needs to be async too
-
-```python
-async def get_source():
-    blah
-    blah
-    await urlopen(url)
-```
+ - token bucket algo to enforce n requests per second
+   - read up on bucket algo types
+ - re-structuring AsyncCrawler to be more testable
+ - use exponential backoff algo?

requirements.txt

@@ -1,8 +1,12 @@
+aiohttp==3.4.4
+async-timeout==3.0.0
+attrs==18.2.0
 beautifulsoup4==4.6.3
 bs4==0.0.1
-certifi==2018.8.13
 chardet==3.0.4
 idna==2.7
 Jinja2==2.10
 lxml==4.2.4
 MarkupSafe==1.0
+multidict==4.4.0
+yarl==1.2.6

templates/sitemap.html.j2

@@ -4,7 +4,7 @@
 </head>
 <body>
 <p>
-Crawled {{ urlcount }} URLs on {{ base_url }} in ~{{ runtime }} seconds.
+Crawled {{ urlcount }} URLs on {{ rooturl }} in ~{{ runtime }} seconds.
 <ul>
 {% for url in urls %}
   <li><a href="{{ url }}">{{ url }}</a></li>

test_helpers.py

@@ -1,34 +1,65 @@
 #!/usr/bin/env python
 
 import unittest
-from utils.helpers import (sanitise_url)
+from unittest import mock
+from utils.helpers import RobotsTxt, standardise_url
+
+
+class TestRobots(unittest.TestCase):
+
+    rooturl = 'http://eu.httpbin.org'
+    no_robots = 'https://www.simonweald.com'
+
+    test_paths = (('/', True), ('/deny', False))
+
+    robots = RobotsTxt(rooturl=rooturl)
+    norobots = RobotsTxt(rooturl=no_robots)
+
+    def test_robots_txt_deny(self):
+        '''
+        Asserts result is True or False.
+        '''
+        for path, allowed in self.test_paths:
+            result = self.robots.check(url=path)
+            self.assertIs(result, allowed)
+
+    def test_no_robots_txt(self):
+        '''
+        Ensure we can crawl if robots.txt isn't present.
+        '''
+        result = self.norobots.check(url='/')
+        self.assertTrue(result)
+
 
 class TestUrls(unittest.TestCase):
 
-    base_url_list = (('eu.httpbin.org', 'http://eu.httpbin.org'),
-                     ('www.simonweald.com', 'http://www.simonweald.com'),
-                     ('http://www.github.com/', 'http://www.github.com'),
-                     ('https://www.github.com', 'https://www.github.com'))
+    rooturl = 'http://eu.httpbin.org'
 
-    urls_to_clean = (('https://www.github.com/', 'https://www.github.com/'),
-                     ('https://github.com/?foo=bar', 'https://github.com/'),
-                     ('https://github.com/#anchor', 'https://github.com/'))
+    rooturl_list = (('eu.httpbin.org', 'http://eu.httpbin.org'),
+                     ('http://eu.httpbin.org/', 'http://eu.httpbin.org'),
+                     ('https://eu.httpbin.org', 'https://eu.httpbin.org'))
 
+    urls_to_clean = (('http://eu.httpbin.org', 'http://eu.httpbin.org'),
+                     ('http://eu.httpbin.org/some/path/', 'http://eu.httpbin.org/some/path/'),
+                     ('http://eu.httpbin.org/index.html','http://eu.httpbin.org/index.html'),
+                     ('http://eu.httpbin.org/index.html?foo=bar', 'http://eu.httpbin.org/index.html'),
+                     ('http://eu.httpbin.org/index.html#anchor', 'http://eu.httpbin.org/index.html'))
 
-    def test_sanitise_base_url(self):
+    def test_standardise_rooturl(self):
         '''
-        Tests whether a URL's protocol can be discovered if not provided.
+        Tests whether a base URL can be standardised to the format
+        proto://[sub].domain.tld.
         '''
-        for url, target in self.base_url_list:
-            result = sanitise_url(url, base_url=True)
+        for url, target in self.rooturl_list:
+            result = standardise_url(url)
             self.assertEqual(result, target)
 
-    def test_sanitise_url(self):
+    def test_standardise_url(self):
         '''
-        Tests whether a URL's protocol can be discovered if not provided.
+        Ensure that fragments/anchors etc are stripped.
         '''
         for url, target in self.urls_to_clean:
-            result = sanitise_url(url)
+            result = standardise_url(url, rooturl=self.rooturl)
             self.assertEqual(result, target)
 
 

utils/helpers.py

@@ -3,153 +3,180 @@
 Utilities to provide various misc functions.
 '''
 
-from bs4 import BeautifulSoup
 import aiohttp
+import asyncio
+from bs4 import BeautifulSoup
+import urllib.error
+from urllib.parse import urljoin, urlsplit
 import urllib.request
 import urllib.robotparser
-import urllib.error
-import gzip
-from urllib.parse import (urljoin, urlsplit)
 
 
-class UrlPool(object):
+class AsyncCrawler(object):
     '''
-    Object to manage a pool of URLs.
+    A concurrent recursive web crawler.
+
+    A recursive web crawler which finds all URLs local to the domains specified
+    in the `rooturl` argument.
+
+    Arguments:
+        rooturl:        Root domain to begin crawling.
+        robots:         RobotsTxt object for the rooturl.
+        concurrency:    number of concurrent pages to crawl.
+
+    Returns:
+        All discovered pages in a set.
     '''
 
-    def __init__(self):
-        self.pool = set()
-
-    def check_duplicate(self, new_url):
-        '''
-        Checks if a URL exists in the current pool.
-        '''
-        if new_url in self.pool:
-            return True
-        else:
-            return False
-
-    def remove_from_pool(self):
-        '''
-        Remove a URL from the pool and return it to be crawled.
-        '''
-        return(self.pool.pop())
-
-    def add_to_pool(self, url):
-        self.pool.add(url)
-
-    def list_pool(self):
-        pool = self.pool
-        return pool
-
-
-class WebPage(object):
-    '''
-    Object to manage common operations required to return
-    the data from each individual page.
-    '''
-
-    # set a sane user-agent and request compression if available.
-    headers = {'Accept-Encoding': 'gzip, deflate',
-               'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
-
-    def __init__(self, url=None, base_url=None, robots=None):
-        self.url = url
-        self.base_url = base_url
+    def __init__(self, rooturl=None, robots=None, concurrency=None):
+        self.rooturl = rooturl
         self.robots = robots
-        self.source = None
-        self.urls_to_crawl = set()
+        self.crawled = set()
+        self.headers = {'Accept-Encoding': 'gzip, deflate',
+                        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
+        self.client_session = None
+        self.semaphore = asyncio.BoundedSemaphore(concurrency)
 
-
-    def get_source(self):
+    async def crawl_url(self, url=None):
         '''
-        Retrieve a page's source.
+        Crawls the given URL and finds all new URLs in the given page.
         '''
+        urls = []
+        source = await self.get_source(url)
+        if source:
+            urls = self.find_all_urls(source)
 
-        request = urllib.request.Request(self.url, headers=self.headers)
-        page = urllib.request.urlopen(request, timeout=5)
+        return urls
 
-        # handle the content encoding in case it needs decompressing.
-        if 'text/html' in page.info().get('Content-Type'):
-            if page.info().get('Content-Encoding'):
-                if page.info().get('Content-Encoding') == 'gzip':
-                    self.source = gzip.decompress(page.read())
-                elif page.info().get('Content-Encoding') == 'deflate':
-                    self.source = page.read()
-            else:
-                self.source = page.read()
-
-
-    def find_links(self):
+    def validate_url(self, url=None):
         '''
-        Find all URLs on a page and ensure they are absolute. If they are
-        relative then they will be appended to the base URL.
+        Ensures we have a valid URL to crawl and that the site's robots.txt
+        allows it.
         '''
-        hrefs = set()
+        # ensure the URL is in a sane format
+        url = standardise_url(url=url, rooturl=self.rooturl)
 
-        soup = BeautifulSoup(self.source, 'lxml')
-        links = soup.find_all('a', href=True)
-
-        for link in links:
-            if link['href'].startswith('/'):
-                hrefs.add(urljoin(self.url, link['href']))
-            else:
-                hrefs.add(link['href'])
-
-        self.discovered_hrefs = hrefs
-
-
-    def parse_urls(self):
-        '''
-        Iterate through the list of discovered URLs and add them to the
-        pool if they start with the base URL.
-        '''
-        for url in self.discovered_hrefs:
-            if url.startswith(self.base_url) and self.robots.check(url):
-                sanitised_url = sanitise_url(url=url)
-                self.urls_to_crawl.add(sanitised_url)
-
-
-    def list_urls(self):
-        '''
-        Returns all valid discovered URLs.
-        '''
-
-        return self.urls_to_crawl
-
-
-    def run(self):
-        '''
-        Attempt to get the page's source and if successful, iterate through it
-        to find any links we can crawl.
-        '''
-        try:
-            self.get_source()
-        except Exception:
-            # skip if we didn't retrieve the source.
-            pass
-
-        if self.source:
-            self.find_links()
-            self.parse_urls()
-            return True
+        if url and self.robots.check(url=url):
+            return url
         else:
             return False
 
+    async def get_source(self, url=None):
+        '''
+        Obtains the URL's source, provided it is HTML. Usage of semaphores
+        ensures only a certain number of coroutines can run at any given
+        time.
+        '''
+        async with self.semaphore:
+            async with self.client_session.head(url, timeout=5) as head:
+                try:
+                    _ = await head.read()
+                except Exception:
+                    pass
+            if 'text/html' in head.headers['Content-Type']:
+                async with self.client_session.get(url, timeout=5) as resp:
+                    try:
+                        source = await resp.read()
+                        return source
+                    except Exception:
+                        return None
+            else:
+                return None
+
+    def find_all_urls(self, source=None):
+        '''
+        Find all URLs in a page's source. Returns a list of URLs which have
+        been validated as local to the starting URL.
+        '''
+        urls = []
+
+        html = BeautifulSoup(source, 'lxml')
+        hrefs = html.find_all('a', href=True)
+
+        # build a set of URLs which are valid and haven't been crawled yet
+        for href in hrefs:
+            url = self.validate_url(url=href['href'])
+            if url and url not in self.crawled:
+                urls.append(url)
+
+        return urls
+
+    async def run(self, urls=None):
+        '''
+        Crawls a batch of URLs of any size (resource usage is bounded by n
+        semaphores (where n = concurrency). Returns a set of URLs to be added
+        to the list of URLs which need to be crawled (find_all_urls only
+        returns unseen URLs).
+        '''
+        tasks = []
+        all_urls = set()
+        for url in urls:
+            # mark the URL as seen.
+            self.crawled.add(url)
+            # create a task to crawl the URL.
+            tasks.append(self.crawl_url(url))
+
+        # wait for all tasks to complete.
+        for task in asyncio.as_completed(tasks):
+            urls = None
+            try:
+                # try getting all tasks as completed.
+                urls = await task
+            except Exception:
+                # skip until all tasks have completed.
+                pass
+
+            # add the URLs to a set to be returned.
+            if urls:
+                for url in urls:
+                    all_urls.add(url)
+
+        return all_urls
+
+    async def main(self):
+        '''
+        Runs a crawl with batches of URLs. Once complete returns a list of all
+        crawled URLs.
+        '''
+        self.client_session = aiohttp.ClientSession(headers=self.headers)
+        to_crawl = []
+        # add the root URL to initialise the crawler.
+        to_crawl.append(self.rooturl)
+
+        print('Crawling: {0}'.format(self.rooturl))
+        while len(to_crawl) > 0:
+            discovered_urls = await self.run(urls=to_crawl)
+            # empty to_crawl list and then add all newly discovered URLs for
+            # the next iteration.
+            to_crawl.clear()
+            to_crawl.extend(discovered_urls)
+            print('{0} URLs crawled'.format(len(self.crawled)))
+
+        # close the ssions once all URLs have been crawled.
+        await self.client_session.close()
+
+        return self.crawled
+
 
 class RobotsTxt(object):
     '''
-    needs a docstring
+    Retrieve and query robots.txt for a given domain.
+
+    Retrieves and parses robots.txt for the given domain. Calling the check()
+    method returns True or False depending on whether crawling of that given
+    URL is allowed.
     '''
 
-    def __init__(self, base_url=None):
+    def __init__(self, rooturl=None):
         '''
-        Manually retrieve robots.txt to allow us to set the user-agent.
+        Manually retrieve robots.txt to allow us to set the user-agent (works
+        around sites which disallow access to robots.txt without a sane
+        user-agent).
         '''
-        self.base_url = base_url
+        self.rooturl = rooturl
         self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
 
-        robots_url = urljoin(self.base_url, 'robots.txt')
+        robots_url = urljoin(self.rooturl, 'robots.txt')
         request = urllib.request.Request(robots_url, headers=self.headers)
 
         robots = urllib.robotparser.RobotFileParser()
@@ -158,6 +185,7 @@ class RobotsTxt(object):
         try:
             response = urllib.request.urlopen(request, timeout=5)
         except urllib.error.HTTPError:
+            # if robots.txt doesn't exist then allow all URLs to be crawled.
             robots.allow_all = True
         else:
             data = response.read()
@@ -166,7 +194,6 @@ class RobotsTxt(object):
 
         self.robots = robots
 
-
     def check(self, url):
         '''
         Test if robots allows us to crawl that URL.
@@ -174,31 +201,40 @@ class RobotsTxt(object):
         return self.robots.can_fetch("*", url)
 
 
-def sanitise_url(url, base_url=False):
+def standardise_url(url=None, rooturl=None):
     '''
-    If `base_url` is True, we attempt to standardise `url` to ensure it can be
-    prepended to relative URLs. If no scheme has been provided then we default
-    to http as any sane https-only site should 301 redirect http > https.
+    If `rooturl` is None then we attempt to standarise the URL to ensure it
+    can be prepended to relative URLs. If no scheme has been provided then we
+    default to http as any sane https-only site should 301 redirect http to
+    https.
 
-    If `base_url` is False, we sanitise URLs to strip queries and fragments (we
-    don't want to scrape in-page anchors etc).
+    If `rooturl` is set, we standardise URLs to strip queries and fragments
+    (we don't want to scrape in-page anchors etc). Any relative URLs will be
+    appended to the root url.
 
-    Returns a sanitised URL as a string.
+    Returns a standardised URL as a string.
     '''
     default_proto = 'http'
     delim = '://'
+    file_extensions = ('/', 'htm', 'html', 'xhtml', 'php', 'asp', 'aspx',
+                       'cfm')
 
     split_url = urlsplit(url)
 
-    if base_url:
-        # This will sanitise the initial url for the initial page crawl.
+    if not rooturl:
+        # This will sanitise the initial url provided by the user.
         if split_url.scheme and split_url.scheme.startswith('http'):
-            sanitised_url = "".join([split_url.scheme, delim, split_url.netloc])
+            return "".join([split_url.scheme, delim, split_url.netloc])
         elif (split_url.path and not split_url.scheme and not split_url.netloc):
-            sanitised_url = "".join([default_proto, delim, split_url.path])
+            return "".join([default_proto, delim, split_url.path])
     else:
+        # if url.endswith(file_extensions):
         # Sanitise discovered URLs. We already expect them in the format
-        # protocol://base_url/path
-        sanitised_url = "".join([split_url.scheme, delim, split_url.netloc, split_url.path])
+        # protocol://rooturl/path
+        if url.startswith('/'):
+            return urljoin(rooturl, split_url.path)
+        elif url.startswith(rooturl):
+            return "".join([split_url.scheme, delim, split_url.netloc,
+                            split_url.path])
 
-    return sanitised_url
+    return None