misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rename all instances of base_url to rooturl, add more documentation

Browse Source
This commit is contained in:
Simon Weald
2018-09-18 18:24:15 +01:00
parent 32d7f1e54b
commit 679b1b7b53
3 changed files with 65 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
async_crawler.py
View File

@@ -8,7 +8,7 @@ URLs discovered will be crawled.
The crawler takes a total of two arguments (concurrency is optional): The crawler takes a total of two arguments (concurrency is optional):
url: the base URL to begin the crawl from. url: the root URL to begin the crawl from.
concurrency: the maximum number of pages which may be crawled concurrently. concurrency: the maximum number of pages which may be crawled concurrently.
''' '''
@@ -29,32 +29,31 @@ def sanity_checks(url=None):
url: the root URL to be crawled. url: the root URL to be crawled.
Returns: Returns:
baseurl: a validated and cleaned version of the initial URL. rooturl: a string containing avalidated and cleaned version of the
(type=string) initial URL.
robots: an object which allows us to query whether a site may be crawled. robots: an object which allows us to query whether a site may be crawled.
(type=RobotsTxt)
''' '''
# ensure we have a sensible URL to work with # ensure we have a sensible URL to work with
baseurl = standardise_url(url=url) rooturl = standardise_url(url=url)
# get robots.txt # get robots.txt
robots = RobotsTxt(base_url=baseurl) robots = RobotsTxt(rooturl=rooturl)
# fail early if robots denies all crawling # fail early if robots denies all crawling
if not robots.check(url=baseurl): if not robots.check(url=rooturl):
sys.exit("{baseurl} cannot be crawled (denied by robots.txt)".format( sys.exit("{0} cannot be crawled (denied by robots.txt)".format(
baseurl=baseurl)) rooturl))
return(baseurl, robots) return(rooturl, robots)
def render_sitemap(base_url=None, crawled_urls=None, runtime=None): def render_sitemap(rooturl=None, crawled_urls=None, runtime=None):
''' '''
Renders the sitemap to an HTML file. Renders the sitemap to an HTML file.
Accepts: Accepts:
base_url: rooturl: string containing the root URL
crawled_urls: crawled_urls: set containing discovered URLs
runtime: runtime: int representing run time of AsyncCrawler
''' '''
urlcount = len(crawled_urls) urlcount = len(crawled_urls)
sorted_urls = sorted(crawled_urls) sorted_urls = sorted(crawled_urls)
@@ -63,7 +62,7 @@ def render_sitemap(base_url=None, crawled_urls=None, runtime=None):
loader=jinja2.FileSystemLoader('templates') loader=jinja2.FileSystemLoader('templates')
).get_template('sitemap.html.j2') ).get_template('sitemap.html.j2')
rendered_html = template.render(base_url=base_url, urlcount=urlcount, rendered_html = template.render(rooturl=rooturl, urlcount=urlcount,
urls=sorted_urls, runtime=runtime) urls=sorted_urls, runtime=runtime)
with open('sitemap.html', 'w') as outfile: with open('sitemap.html', 'w') as outfile:
@@ -79,10 +78,10 @@ def main():
''' '''
starttime = datetime.now() starttime = datetime.now()
baseurl, robots = sanity_checks(url=args.url) rooturl, robots = sanity_checks(url=args.url)
# create a crawler # create a crawler
async_crawler = AsyncCrawler(baseurl=baseurl, robots=robots, async_crawler = AsyncCrawler(rooturl=rooturl, robots=robots,
concurrency=args.concurrency) concurrency=args.concurrency)
# create a task to run the crawler, run the loop and then gather the # create a task to run the crawler, run the loop and then gather the
@@ -95,7 +94,7 @@ def main():
runtime = int((datetime.now() - starttime).total_seconds()) runtime = int((datetime.now() - starttime).total_seconds())
render_sitemap(base_url=baseurl, crawled_urls=results, runtime=runtime) render_sitemap(rooturl=rooturl, crawled_urls=results, runtime=runtime)
if __name__ == '__main__': if __name__ == '__main__':

2
templates/sitemap.html.j2
View File

@@ -4,7 +4,7 @@
</head> </head>
<body> <body>
<p> <p>
Crawled {{ urlcount }} URLs on {{ base_url }} in ~{{ runtime }} seconds. Crawled {{ urlcount }} URLs on {{ rooturl }} in ~{{ runtime }} seconds.
<ul> <ul>
{% for url in urls %} {% for url in urls %}
<li><a href="{{ url }}">{{ url }}</a></li> <li><a href="{{ url }}">{{ url }}</a></li>

73
utils/helpers.py
View File

@@ -14,11 +14,22 @@ import urllib.robotparser
class AsyncCrawler(object): class AsyncCrawler(object):
''' '''
docstring A concurrent recursive web crawler.
A recursive web crawler which finds all URLs local to the domains specified
in the `rooturl` argument.
Arguments:
rooturl: Root domain to begin crawling.
robots: RobotsTxt object for the rooturl.
concurrency: number of concurrent pages to crawl.
Returns:
All discovered pages in a set.
''' '''
def __init__(self, baseurl=None, robots=None, concurrency=None): def __init__(self, rooturl=None, robots=None, concurrency=None):
self.baseurl = baseurl self.rooturl = rooturl
self.robots = robots self.robots = robots
self.crawled = set() self.crawled = set()
self.headers = {'Accept-Encoding': 'gzip, deflate', self.headers = {'Accept-Encoding': 'gzip, deflate',
@@ -28,7 +39,7 @@ class AsyncCrawler(object):
async def crawl_url(self, url=None): async def crawl_url(self, url=None):
''' '''
Crawls the given URL and finds all new URLs in the initial page. Crawls the given URL and finds all new URLs in the given page.
''' '''
urls = [] urls = []
source = await self.get_source(url) source = await self.get_source(url)
@@ -43,7 +54,7 @@ class AsyncCrawler(object):
allows it. allows it.
''' '''
# ensure the URL is in a sane format # ensure the URL is in a sane format
url = standardise_url(url=url, base_url=self.baseurl) url = standardise_url(url=url, rooturl=self.rooturl)
if url and self.robots.check(url=url): if url and self.robots.check(url=url):
return url return url
@@ -53,14 +64,15 @@ class AsyncCrawler(object):
async def get_source(self, url=None): async def get_source(self, url=None):
''' '''
Obtains the URL's source, provided it is HTML. Usage of semaphores Obtains the URL's source, provided it is HTML. Usage of semaphores
ensures only a certain number of coroutines can run at once. ensures only a certain number of coroutines can run at any given
time.
''' '''
async with self.semaphore: async with self.semaphore:
async with self.client_session.head(url, timeout=5) as head: async with self.client_session.head(url, timeout=5) as head:
try: try:
_ = await head.read() _ = await head.read()
except Exception as e: except Exception:
print(e) pass
if 'text/html' in head.headers['Content-Type']: if 'text/html' in head.headers['Content-Type']:
async with self.client_session.get(url, timeout=5) as resp: async with self.client_session.get(url, timeout=5) as resp:
try: try:
@@ -68,6 +80,8 @@ class AsyncCrawler(object):
return source return source
except Exception: except Exception:
return None return None
else:
return None
def find_all_urls(self, source=None): def find_all_urls(self, source=None):
''' '''
@@ -99,21 +113,22 @@ class AsyncCrawler(object):
for url in urls: for url in urls:
# mark the URL as seen. # mark the URL as seen.
self.crawled.add(url) self.crawled.add(url)
# create an coroutine to crawl the URL. # create a task to crawl the URL.
tasks.append(self.crawl_url(url)) tasks.append(self.crawl_url(url))
# wait for all tasks to complete. # wait for all tasks to complete.
for task in asyncio.as_completed(tasks): for task in asyncio.as_completed(tasks):
urls = None urls = None
try: try:
# try getting all tasks as completed.
urls = await task urls = await task
except Exception as e: except Exception:
print(e) # skip until all tasks have completed.
pass
# add the URLs to a set to be returned. # add the URLs to a set to be returned.
if urls: if urls:
for url in urls: for url in urls:
print('Found: {0}'.format(url))
all_urls.add(url) all_urls.add(url)
return all_urls return all_urls
@@ -125,15 +140,17 @@ class AsyncCrawler(object):
''' '''
self.client_session = aiohttp.ClientSession(headers=self.headers) self.client_session = aiohttp.ClientSession(headers=self.headers)
to_crawl = [] to_crawl = []
to_crawl.append(self.baseurl) # add the root URL to initialise the crawler.
to_crawl.append(self.rooturl)
print('Crawling: {0}'.format(self.baseurl)) print('Crawling: {0}'.format(self.rooturl))
while len(to_crawl) > 0: while len(to_crawl) > 0:
discovered_urls = await self.run(urls=to_crawl) discovered_urls = await self.run(urls=to_crawl)
# empty to_crawl list and then add all newly discovered URLs for # empty to_crawl list and then add all newly discovered URLs for
# the next iteration. # the next iteration.
to_crawl.clear() to_crawl.clear()
to_crawl.extend(discovered_urls) to_crawl.extend(discovered_urls)
print('{0} URLs crawled'.format(len(self.crawled)))
# close the ssions once all URLs have been crawled. # close the ssions once all URLs have been crawled.
await self.client_session.close() await self.client_session.close()
@@ -143,19 +160,23 @@ class AsyncCrawler(object):
class RobotsTxt(object): class RobotsTxt(object):
''' '''
needs a docstring Retrieve and query robots.txt for a given domain.
Retrieves and parses robots.txt for the given domain. Calling the check()
method returns True or False depending on whether crawling of that given
URL is allowed.
''' '''
def __init__(self, base_url=None): def __init__(self, rooturl=None):
''' '''
Manually retrieve robots.txt to allow us to set the user-agent (works Manually retrieve robots.txt to allow us to set the user-agent (works
around sites which disallow access to robots.txt without a sane around sites which disallow access to robots.txt without a sane
user-agent). user-agent).
''' '''
self.base_url = base_url self.rooturl = rooturl
self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'} self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}
robots_url = urljoin(self.base_url, 'robots.txt') robots_url = urljoin(self.rooturl, 'robots.txt')
request = urllib.request.Request(robots_url, headers=self.headers) request = urllib.request.Request(robots_url, headers=self.headers)
robots = urllib.robotparser.RobotFileParser() robots = urllib.robotparser.RobotFileParser()
@@ -180,16 +201,16 @@ class RobotsTxt(object):
return self.robots.can_fetch("*", url) return self.robots.can_fetch("*", url)
def standardise_url(url=None, base_url=None): def standardise_url(url=None, rooturl=None):
''' '''
If `base_url` is None then we attempt to standarise the URL to ensure it If `rooturl` is None then we attempt to standarise the URL to ensure it
can be prepended to relative URLs. If no scheme has been provided then we can be prepended to relative URLs. If no scheme has been provided then we
default to http as any sane https-only site should 301 redirect http to default to http as any sane https-only site should 301 redirect http to
https. https.
If `base_url` is set, we standardise URLs to strip queries and fragments If `rooturl` is set, we standardise URLs to strip queries and fragments
(we don't want to scrape in-page anchors etc). Any relative URLs will be (we don't want to scrape in-page anchors etc). Any relative URLs will be
appended to the base url. appended to the root url.
Returns a standardised URL as a string. Returns a standardised URL as a string.
''' '''
@@ -200,7 +221,7 @@ def standardise_url(url=None, base_url=None):
split_url = urlsplit(url) split_url = urlsplit(url)
if not base_url: if not rooturl:
# This will sanitise the initial url provided by the user. # This will sanitise the initial url provided by the user.
if split_url.scheme and split_url.scheme.startswith('http'): if split_url.scheme and split_url.scheme.startswith('http'):
return "".join([split_url.scheme, delim, split_url.netloc]) return "".join([split_url.scheme, delim, split_url.netloc])
@@ -209,10 +230,10 @@ def standardise_url(url=None, base_url=None):
else: else:
# if url.endswith(file_extensions): # if url.endswith(file_extensions):
# Sanitise discovered URLs. We already expect them in the format # Sanitise discovered URLs. We already expect them in the format
# protocol://base_url/path # protocol://rooturl/path
if url.startswith('/'): if url.startswith('/'):
return urljoin(base_url, split_url.path) return urljoin(rooturl, split_url.path)
elif url.startswith(base_url): elif url.startswith(rooturl):
return "".join([split_url.scheme, delim, split_url.netloc, return "".join([split_url.scheme, delim, split_url.netloc,
split_url.path]) split_url.path])