add test files

.gitignore

@@ -2,3 +2,4 @@
 venv/
 .vscode/*
 __pycache__/
+sitemap.html

README.md

@@ -1 +1,28 @@
-# Concurrent web scraper
+# Concurrent web scraper
+
+## Requirements
+
+This crawler requires at least Python 3.5 in order to utilise the async/await keywords from `asyncio`.
+
+Install required modules:
+
+```bash
+pip install -r requirements.txt
+```
+
+Run:
+
+```bash
+python async_crawler.py -u https://urltocrawl.com [-c 100]
+```
+
+Flags:
+
+  - `-u/--url https://url.com`
+    - The base URL is required.
+  - `-c/--concurrency 100`
+    - Specifying concurrency value is optional (defaults to 100).
+
+## Results
+
+The resulting sitemap will be output to the root of this directory as `sitemap.html`

async_crawler.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+'''
+Asynchronous web crawler written in Python 3.5+.
+
+This script will respect the site's `robots.txt`, if one exists. If not, all
+URLs discovered will be crawled.
+
+The crawler takes a total of two arguments (concurrency is optional):
+
+    url: the root URL to begin the crawl from.
+    concurrency: the maximum number of pages which may be crawled concurrently.
+'''
+
+import argparse
+import asyncio
+from datetime import datetime
+import jinja2
+import os
+import sys
+from utils.helpers import RobotsTxt, AsyncCrawler, standardise_url
+
+
+def sanity_checks(url=None):
+    '''
+    Runs some basic sanity checks before the crawler is initialised.
+
+    Accepts:
+      url: the root URL to be crawled.
+
+    Returns:
+      rooturl: a string containing avalidated and cleaned version of the
+               initial URL.
+      robots: an object which allows us to query whether a site may be crawled.
+    '''
+    # ensure we have a sensible URL to work with
+    rooturl = standardise_url(url=url)
+    # get robots.txt
+    robots = RobotsTxt(rooturl=rooturl)
+
+    # fail early if robots denies all crawling
+    if not robots.check(url=rooturl):
+        sys.exit("{0} cannot be crawled (denied by robots.txt)".format(
+                 rooturl))
+
+    return(rooturl, robots)
+
+
+def render_sitemap(rooturl=None, crawled_urls=None, runtime=None):
+    '''
+    Renders the sitemap to an HTML file.
+
+    Accepts:
+        rooturl: string containing the root URL
+        crawled_urls: set containing discovered URLs
+        runtime: int representing run time of AsyncCrawler
+    '''
+    urlcount = len(crawled_urls)
+    sorted_urls = sorted(crawled_urls)
+
+    template = jinja2.Environment(
+        loader=jinja2.FileSystemLoader('templates')
+    ).get_template('sitemap.html.j2')
+
+    rendered_html = template.render(rooturl=rooturl, urlcount=urlcount,
+                                    urls=sorted_urls, runtime=runtime)
+
+    with open('sitemap.html', 'w') as outfile:
+        outfile.write(rendered_html)
+
+    print('Sitemap available at {0}/sitemap.html'.format(os.getcwd()))
+
+
+def main():
+    '''
+    Main function, responsible for prepping and running the crawler and
+    rendering the sitemap.
+    '''
+    starttime = datetime.now()
+
+    rooturl, robots = sanity_checks(url=args.url)
+
+    # create a crawler
+    async_crawler = AsyncCrawler(rooturl=rooturl, robots=robots,
+                                 concurrency=args.concurrency)
+
+    # create a task to run the crawler, run the loop and then gather the
+    # results.
+    task = asyncio.Task(async_crawler.main())
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(task)
+    loop.close()
+    results = sorted(task.result())
+
+    runtime = int((datetime.now() - starttime).total_seconds())
+
+    render_sitemap(rooturl=rooturl, crawled_urls=results, runtime=runtime)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Recursive web crawler')
+    parser.add_argument("-u", "--url", required=True, help="Initial url")
+    parser.add_argument("-c", "--concurrency", required=False, type=int,
+                        default=100, help="Max pages to crawl concurrently")
+    args = parser.parse_args()
+
+    main()

crawler.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+'''
+Need a docstring.
+'''
+
+import argparse
+import jinja2
+import os
+import asyncio
+from datetime import datetime
+from utils.helpers import (UrlPool, WebPage, RobotsTxt, sanitise_url)
+
+
+def init_crawler(base_url=None, robots=None):
+    '''
+    Initialises the crawler by running the initial URL.
+    '''
+    uncrawled_urls, crawled_urls = UrlPool(), UrlPool()
+    initial_page = WebPage(url=base_url, base_url=base_url, robots=robots)
+
+    try:
+        initial_page.run()
+    except Exception as e:
+        print(e)
+
+    initial_urls = initial_page.list_urls()
+
+    # ensure the base URL isn't crawled again
+    try:
+        initial_urls.remove(base_url)
+    except KeyError:
+        pass
+    # also ensure base URL wasn't discovered with a trailing slash on the
+    # initial page scrape
+    try:
+        initial_urls.remove("".join([base_url, '/']))
+    except KeyError:
+        pass
+
+    # Add the base URL to the crawled pool
+    crawled_urls.add_to_pool(base_url)
+
+    for url in initial_urls:
+        sanitised_url = sanitise_url(url=url)
+        if sanitised_url not in crawled_urls.pool:
+            uncrawled_urls.add_to_pool(sanitised_url)
+
+    return(uncrawled_urls, crawled_urls)
+
+
+def process_pool(base_url=None, uncrawled_urls=None, crawled_urls=None, robots=None):
+    '''
+    Iterates over the pool of URLs and adds any discovered URLs.
+    '''
+    while uncrawled_urls.pool:
+        # pop url from pool
+        new_url = uncrawled_urls.remove_from_pool()
+        # create a WebPage object for the URL
+        current_page = WebPage(url=new_url, base_url=base_url, robots=robots)
+        try:
+            succeeded = current_page.run()
+        except Exception as e:
+            print(e)
+
+        if succeeded:
+            _urls = current_page.list_urls()
+            crawled_urls.add_to_pool(new_url)
+
+            for url in _urls:
+                sanitised_url = sanitise_url(url=url)
+                if sanitised_url not in crawled_urls.pool:
+                    uncrawled_urls.add_to_pool(url)
+
+        print('{0} URLs crawled, {1} remaining'.format(len(crawled_urls.pool),
+                len(uncrawled_urls.pool)))
+
+
+def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
+    '''
+    Renders the sitemap as an HTML file.
+    '''
+    urlcount = len(crawled_urls)
+    sorted_urls = sorted(crawled_urls)
+
+    tmpl = jinja2.Environment(
+        loader=jinja2.FileSystemLoader('templates')
+        ).get_template('sitemap.html.j2')
+
+    rendered_html = tmpl.render(base_url=base_url, urlcount=urlcount, urls=sorted_urls, runtime=runtime)
+
+    with open('sitemap.html', 'w') as outfile:
+        outfile.write(rendered_html)
+
+    print('Sitemap available at {0}/sitemap.html'.format(os.getcwd()))
+
+
+def run(args=None):
+    '''
+    needs a docstring.
+    '''
+    starttime = datetime.now()
+
+    base_url = sanitise_url(args.url, base_url=True)
+    robots = RobotsTxt(base_url=base_url)
+
+    uncrawled_urls, crawled_urls = init_crawler(base_url, robots)
+    process_pool(base_url, uncrawled_urls, crawled_urls, robots)
+
+    runtime = int((datetime.now() - starttime).total_seconds())
+
+    render_sitemap(base_url=base_url, crawled_urls=crawled_urls.pool, runtime=runtime)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Recursive web crawler')
+    parser.add_argument("-u", "--url", required=True, help="Base url to crawl")
+    args = parser.parse_args()
+
+    run(args)

notes.md

@@ -0,0 +1,26 @@
+## Thoughts
+
+  * ~~strip hashes and everything following (as they're in-page anchors)~~
+  * ~~strip args~~
+  * ~~use `pop()` on the set instead of `.remove()`~~
+    * ~~return false once the set is empty~~
+  * ~~`WebPage.parse_urls()` needs to compare startswith to base url~~
+  * ~~ignore any links which aren't to pages~~
+  * ~~better url checking to get bare domain~~ #wontfix
+  * ~~remove trailing slash from any discovered url~~
+  * ~~investigate lxml parser~~
+  * ~~remove base url from initial urls with and without trailing slash~~
+  * ~~investigate using [tldextract](https://github.com/john-kurkowski/tldextract) to match urls~~ #wontfix
+  * ~~implement parsing of [robots.txt](http://docs.w3cub.com/python~3.6/library/urllib.robotparser/)~~
+  * ~~investigate [gzip encoding](https://stackoverflow.com/questions/36383227/avoid-downloading-images-using-beautifulsoup-and-urllib-request)~~
+  * ~~implement some kind of progress display~~
+  * async
+  * better exception handling
+  * randomise output filename
+
+### talking points
+
+ - token bucket algo to enforce n requests per second
+   - read up on bucket algo types
+ - re-structuring AsyncCrawler to be more testable
+ - use exponential backoff algo?

requirements.txt

@@ -1,7 +1,12 @@
+aiohttp==3.4.4
+async-timeout==3.0.0
+attrs==18.2.0
 beautifulsoup4==4.6.3
 bs4==0.0.1
-certifi==2018.8.13
 chardet==3.0.4
 idna==2.7
-requests==2.19.1
-urllib3==1.23
+Jinja2==2.10
+lxml==4.2.4
+MarkupSafe==1.0
+multidict==4.4.0
+yarl==1.2.6

scraper.py

@@ -1 +0,0 @@
-#!/usr/bin/env python

templates/sitemap.html.j2

@@ -0,0 +1,14 @@
+<html>
+<head>
+ <title>Sitemap for {{ base_url }}</title>
+</head>
+<body>
+<p>
+Crawled {{ urlcount }} URLs on {{ rooturl }} in ~{{ runtime }} seconds.
+<ul>
+{% for url in urls %}
+  <li><a href="{{ url }}">{{ url }}</a></li>
+{% endfor %}
+</ul>
+</body>
+</html>

test/files/find_all_urls.html

@@ -0,0 +1,10 @@
+<html>
+<body>
+<p>
+<ul>
+  <li><a href="http://eu.httpbin.org/a/">http://eu.httpbin.org/a/</a></li>
+  <li><a href="http://eu.httpbin.org/b/">http://eu.httpbin.org/b/</a></li>
+  <li><a href="http://eu.httpbin.org/c/">http://eu.httpbin.org/c/</a></li>
+</ul>
+</body>
+</html>

test_helpers.py

@@ -1,37 +1,66 @@
 #!/usr/bin/env python
 
 import unittest
-from utils.helpers import (clean_base_url)
+from unittest import mock
+from utils.helpers import RobotsTxt, standardise_url
+
+
+class TestRobots(unittest.TestCase):
+
+    rooturl = 'http://eu.httpbin.org'
+    no_robots = 'https://www.simonweald.com'
+
+    test_paths = (('/', True), ('/deny', False))
+
+    robots = RobotsTxt(rooturl=rooturl)
+    norobots = RobotsTxt(rooturl=no_robots)
+
+    def test_robots_txt_deny(self):
+        '''
+        Asserts result is True or False.
+        '''
+        for path, allowed in self.test_paths:
+            result = self.robots.check(url=path)
+            self.assertIs(result, allowed)
+
+    def test_no_robots_txt(self):
+        '''
+        Ensure we can crawl if robots.txt isn't present.
+        '''
+        result = self.norobots.check(url='/')
+        self.assertTrue(result)
+
 
 class TestUrls(unittest.TestCase):
 
-    base_url = "github.com"
+    rooturl = 'http://eu.httpbin.org'
 
-    base_url_list = (('eu.httpbin.org', 'http://eu.httpbin.org'),
-                     ('www.simonweald.com', 'http://www.simonweald.com'),
-                     ('http://www.github.com/', 'http://www.github.com'),
-                     ('https://www.github.com', 'https://www.github.com'))
+    rooturl_list = (('eu.httpbin.org', 'http://eu.httpbin.org'),
+                     ('http://eu.httpbin.org/', 'http://eu.httpbin.org'),
+                     ('https://eu.httpbin.org', 'https://eu.httpbin.org'))
 
-    valid_urls = ["https://www.github.com", "http://www.github.com",
-                  "github.com", "/some/url/", "index.html"]
+    urls_to_clean = (('http://eu.httpbin.org', 'http://eu.httpbin.org'),
+                     ('http://eu.httpbin.org/some/path/', 'http://eu.httpbin.org/some/path/'),
+                     ('http://eu.httpbin.org/index.html','http://eu.httpbin.org/index.html'),
+                     ('http://eu.httpbin.org/index.html?foo=bar', 'http://eu.httpbin.org/index.html'),
+                     ('http://eu.httpbin.org/index.html#anchor', 'http://eu.httpbin.org/index.html'))
 
-
-    def test_clean_base_url(self):
+    def test_standardise_rooturl(self):
         '''
-        Tests whether a URL's protocol can be discovered if not provided.
+        Tests whether a base URL can be standardised to the format
+        proto://[sub].domain.tld.
         '''
-        for url, target in self.base_url_list:
-            result = clean_base_url(url)
+        for url, target in self.rooturl_list:
+            result = standardise_url(url)
             self.assertEqual(result, target)
 
-    # def test_url_validation(self):
-    #     '''
-    #     Passes when given a valid URL. A valid URL is qualified
-    #     by being local to the domain to be crawled.
-    #     '''
-    #     for url in self.valid_urls:
-    #         result = url_validation(self.base_url, url)
-    #         self.assertTrue(result)
+    def test_standardise_url(self):
+        '''
+        Ensure that fragments/anchors etc are stripped.
+        '''
+        for url, target in self.urls_to_clean:
+            result = standardise_url(url, rooturl=self.rooturl)
+            self.assertEqual(result, target)
 
 
 if __name__ == '__main__':

utils/helpers.py

@@ -3,36 +3,238 @@
 Utilities to provide various misc functions.
 '''
 
+import aiohttp
+import asyncio
+from bs4 import BeautifulSoup
+import urllib.error
+from urllib.parse import urljoin, urlsplit
+import urllib.request
+import urllib.robotparser
 
-def clean_base_url(url):
-    '''
-    Standardise the URL to be scraped to ensure it
-    is added to relative URLs in a consistent manner.
-    '''
-    protocol = 'http://'
 
-    if url.startswith('http'):
-        base_url = url
+class AsyncCrawler(object):
+    '''
+    A concurrent recursive web crawler.
+
+    A recursive web crawler which finds all URLs local to the domains specified
+    in the `rooturl` argument.
+
+    Arguments:
+        rooturl:        Root domain to begin crawling.
+        robots:         RobotsTxt object for the rooturl.
+        concurrency:    number of concurrent pages to crawl.
+
+    Returns:
+        All discovered pages in a set.
+    '''
+
+    def __init__(self, rooturl=None, robots=None, concurrency=None):
+        self.rooturl = rooturl
+        self.robots = robots
+        self.crawled = set()
+        self.headers = {'Accept-Encoding': 'gzip, deflate',
+                        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
+        self.client_session = None
+        self.semaphore = asyncio.BoundedSemaphore(concurrency)
+
+    async def crawl_url(self, url=None):
+        '''
+        Crawls the given URL and finds all new URLs in the given page.
+        '''
+        urls = []
+        source = await self.get_source(url)
+        if source:
+            urls = self.find_all_urls(source)
+
+        return urls
+
+    def validate_url(self, url=None):
+        '''
+        Ensures we have a valid URL to crawl and that the site's robots.txt
+        allows it.
+        '''
+        # ensure the URL is in a sane format
+        url = standardise_url(url=url, rooturl=self.rooturl)
+
+        if url and self.robots.check(url=url):
+            return url
+        else:
+            return False
+
+    async def get_source(self, url=None):
+        '''
+        Obtains the URL's source, provided it is HTML. Usage of semaphores
+        ensures only a certain number of coroutines can run at any given
+        time.
+        '''
+        async with self.semaphore:
+            async with self.client_session.head(url, timeout=5) as head:
+                try:
+                    _ = await head.read()
+                except Exception:
+                    pass
+            if 'text/html' in head.headers['Content-Type']:
+                async with self.client_session.get(url, timeout=5) as resp:
+                    try:
+                        source = await resp.read()
+                        return source
+                    except Exception:
+                        return None
+            else:
+                return None
+
+    def find_all_urls(self, source=None):
+        '''
+        Find all URLs in a page's source. Returns a list of URLs which have
+        been validated as local to the starting URL.
+        '''
+        urls = []
+
+        html = BeautifulSoup(source, 'lxml')
+        hrefs = html.find_all('a', href=True)
+
+        # build a set of URLs which are valid and haven't been crawled yet
+        for href in hrefs:
+            url = self.validate_url(url=href['href'])
+            if url and url not in self.crawled:
+                urls.append(url)
+
+        return urls
+
+    async def run(self, urls=None):
+        '''
+        Crawls a batch of URLs of any size (resource usage is bounded by n
+        semaphores (where n = concurrency). Returns a set of URLs to be added
+        to the list of URLs which need to be crawled (find_all_urls only
+        returns unseen URLs).
+        '''
+        tasks = []
+        all_urls = set()
+        for url in urls:
+            # mark the URL as seen.
+            self.crawled.add(url)
+            # create a task to crawl the URL.
+            tasks.append(self.crawl_url(url))
+
+        # wait for all tasks to complete.
+        for task in asyncio.as_completed(tasks):
+            urls = None
+            try:
+                # try getting all tasks as completed.
+                urls = await task
+            except Exception:
+                # skip until all tasks have completed.
+                pass
+
+            # add the URLs to a set to be returned.
+            if urls:
+                for url in urls:
+                    all_urls.add(url)
+
+        return all_urls
+
+    async def main(self):
+        '''
+        Runs a crawl with batches of URLs. Once complete returns a list of all
+        crawled URLs.
+        '''
+        self.client_session = aiohttp.ClientSession(headers=self.headers)
+        to_crawl = []
+        # add the root URL to initialise the crawler.
+        to_crawl.append(self.rooturl)
+
+        print('Crawling: {0}'.format(self.rooturl))
+        while len(to_crawl) > 0:
+            discovered_urls = await self.run(urls=to_crawl)
+            # empty to_crawl list and then add all newly discovered URLs for
+            # the next iteration.
+            to_crawl.clear()
+            to_crawl.extend(discovered_urls)
+            print('{0} URLs crawled'.format(len(self.crawled)))
+
+        # close the ssions once all URLs have been crawled.
+        await self.client_session.close()
+
+        return self.crawled
+
+
+class RobotsTxt(object):
+    '''
+    Retrieve and query robots.txt for a given domain.
+
+    Retrieves and parses robots.txt for the given domain. Calling the check()
+    method returns True or False depending on whether crawling of that given
+    URL is allowed.
+    '''
+
+    def __init__(self, rooturl=None):
+        '''
+        Manually retrieve robots.txt to allow us to set the user-agent (works
+        around sites which disallow access to robots.txt without a sane
+        user-agent).
+        '''
+        self.rooturl = rooturl
+        self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
+
+        robots_url = urljoin(self.rooturl, 'robots.txt')
+        request = urllib.request.Request(robots_url, headers=self.headers)
+
+        robots = urllib.robotparser.RobotFileParser()
+        robots.set_url(robots_url)
+
+        try:
+            response = urllib.request.urlopen(request, timeout=5)
+        except urllib.error.HTTPError:
+            # if robots.txt doesn't exist then allow all URLs to be crawled.
+            robots.allow_all = True
+        else:
+            data = response.read()
+            decoded_data = data.decode("utf-8").splitlines()
+            robots.parse(decoded_data)
+
+        self.robots = robots
+
+    def check(self, url):
+        '''
+        Test if robots allows us to crawl that URL.
+        '''
+        return self.robots.can_fetch("*", url)
+
+
+def standardise_url(url=None, rooturl=None):
+    '''
+    If `rooturl` is None then we attempt to standarise the URL to ensure it
+    can be prepended to relative URLs. If no scheme has been provided then we
+    default to http as any sane https-only site should 301 redirect http to
+    https.
+
+    If `rooturl` is set, we standardise URLs to strip queries and fragments
+    (we don't want to scrape in-page anchors etc). Any relative URLs will be
+    appended to the root url.
+
+    Returns a standardised URL as a string.
+    '''
+    default_proto = 'http'
+    delim = '://'
+    file_extensions = ('/', 'htm', 'html', 'xhtml', 'php', 'asp', 'aspx',
+                       'cfm')
+
+    split_url = urlsplit(url)
+
+    if not rooturl:
+        # This will sanitise the initial url provided by the user.
+        if split_url.scheme and split_url.scheme.startswith('http'):
+            return "".join([split_url.scheme, delim, split_url.netloc])
+        elif (split_url.path and not split_url.scheme and not split_url.netloc):
+            return "".join([default_proto, delim, split_url.path])
     else:
-        # otherwise assume HTTP as any sane site should upgrade
-        # to HTTPS via a 301 redirect.
-        base_url = "".join([protocol, url])
+        # if url.endswith(file_extensions):
+        # Sanitise discovered URLs. We already expect them in the format
+        # protocol://rooturl/path
+        if url.startswith('/'):
+            return urljoin(rooturl, split_url.path)
+        elif url.startswith(rooturl):
+            return "".join([split_url.scheme, delim, split_url.netloc,
+                            split_url.path])
 
-    # strip the trailing slash to allow us to append
-    # relative URLs.
-    if base_url.endswith('/'):
-        base_url = base_url[:-1]
-
-    return base_url
-
-
-# def get_url_validation(base_url=None, url=None):
-#     '''
-#     Checks if a URL is valid. Can be absolute or relative.
-#     '''
-
-#     if url.startswith('/'):
-#         full_url = '{0}{1}'.format(base_url, url)
-#     if url.startswith(ffbase_url):
-#         full_url = url
-#     elif url.startswith('/'):
+    return None