small improvements to docs and variables

This commit is contained in:
2018-09-17 21:44:04 +01:00
parent eb2395d461
commit 73c21e5bd3
2 changed files with 16 additions and 2 deletions

View File

@@ -58,7 +58,7 @@ class AsyncCrawler(object):
async with self.semaphore:
async with self.client_session.head(url, timeout=5) as head:
try:
data = await head.read()
_ = await head.read()
except Exception as e:
print(e)
if 'text/html' in head.headers['Content-Type']:
@@ -130,7 +130,7 @@ class AsyncCrawler(object):
print('Crawling: {0}'.format(self.baseurl))
while len(to_crawl) > 0:
discovered_urls = await self.run(urls=to_crawl)
# empty toe crawl list and then add all newly discovered URLs for
# empty to_crawl list and then add all newly discovered URLs for
# the next iteration.
to_crawl.clear()
to_crawl.extend(discovered_urls)