commit of working async crawler

utils/helpers.py

@@ -3,10 +3,10 @@
 Utilities to provide various misc functions.
 '''
 
-import urllib.request
-import urllib.error
-import gzip
-
+# import urllib.request
+# import urllib.error
+# import gzip
+# from time import sleep
 
 
 import aiohttp
@@ -26,9 +26,8 @@ class AsyncCrawler(object):
         self.robots = robots
         self.uncrawled = set()
         self.crawled = set()
-        # self.headers = {'Accept-Encoding': 'gzip, deflate',
-                        # 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
-        self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
+        self.headers = {'Accept-Encoding': 'gzip, deflate',
+                        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
         self.client_session = None
         self.semaphore = asyncio.BoundedSemaphore(concurrency)
 
@@ -37,13 +36,10 @@ class AsyncCrawler(object):
         '''
         docstring
         '''
-        urls = set()
+        urls = []
         source = await self.get_source(url)
         if source:
-            # add the URL we've just crawled
-            self.crawled.add(url)
-            for new_url in self.find_all_urls(source):
-                urls.add(new_url)
+            urls = self.find_all_urls(source)
 
         return urls
 
@@ -66,21 +62,29 @@ class AsyncCrawler(object):
         '''
         Obtains the page's source.
         '''
-        print('semaphore held for {0}'.format(url))
         async with self.semaphore:
-            async with self.client_session.get(url, timeout=5) as resp:
+            async with self.client_session.head(url, timeout=5) as head:
                 try:
-                    source = await resp.read()
-                    return source
-                except Exception:
-                    return None
+                    data = await head.read()
+                except Exception as e:
+                    print(e)
+            if 'text/html' in head.headers['Content-Type']:
+                async with self.client_session.get(url, timeout=5) as resp:
+                    try:
+                        source = await resp.read()
+                        print('crawled {0}'.format(url))
+                        return source
+                    except Exception:
+                        return None
+            else:
+                print('{0} - {1}'.format(head.headers['Content-Type'], url))
 
 
     def find_all_urls(self, source=None):
         '''
         Find all URLs in a page's source.
         '''
-        urls = set()
+        urls = []
 
         html = BeautifulSoup(source, 'lxml')
         hrefs = html.find_all('a', href=True)
@@ -89,27 +93,45 @@ class AsyncCrawler(object):
         for href in hrefs:
             url = self.validate_url(url=href['href'])
             if url and url not in self.crawled:
-                urls.add(url)
+                urls.append(url)
 
         return urls
 
 
-    async def run_loop(self):
-        '''
-        function which runs the crawler
-        '''
-        # print('Crawling: {}'.format(self.baseurl))
-        self.client_session = aiohttp.ClientSession(headers=self.headers)
-        # provide the starting URL to the crawler
-        self.uncrawled.add(self.baseurl)
+    async def run(self, urls=None):
+        tasks = []
+        all_urls = set()
+        for url in urls:
+            self.crawled.add(url)
+            tasks.append(self.crawl_url(url))
 
-        while len(self.uncrawled) > 0:
-            url = self.uncrawled.pop()
-            new_urls = await self.crawl_url(url=url)
-            for url in new_urls:
-                self.uncrawled.add(url)
+        for task in asyncio.as_completed(tasks):
+            urls = None
+            try:
+                # completed.append((await task))
+                urls = await task
+            except Exception as e:
+                print(e)
+
+            if urls:
+                for url in urls:
+                    all_urls.add(url)
+
+        return all_urls
+
+
+    async def main(self):
+        self.client_session = aiohttp.ClientSession(headers=self.headers)
+        to_crawl = []
+        to_crawl.append(self.baseurl)
+
+        while len(to_crawl) > 0:
+            discovered_urls = await self.run(urls=to_crawl)
+            to_crawl.clear()
+            to_crawl.extend(discovered_urls)
 
         await self.client_session.close()
+
         return self.crawled
 
 
@@ -164,6 +186,7 @@ def standardise_url(url=None, base_url=None):
     '''
     default_proto = 'http'
     delim = '://'
+    file_extensions = ('/', 'htm', 'html', 'xhtml', 'php', 'asp', 'aspx', 'cfm')
 
     split_url = urlsplit(url)
 
@@ -174,6 +197,7 @@ def standardise_url(url=None, base_url=None):
         elif (split_url.path and not split_url.scheme and not split_url.netloc):
             return "".join([default_proto, delim, split_url.path])
     else:
+        # if url.endswith(file_extensions):
         # Sanitise discovered URLs. We already expect them in the format
         # protocol://base_url/path
         if url.startswith('/'):