small improvements to docs and variables
This commit is contained in:
@@ -58,7 +58,7 @@ class AsyncCrawler(object):
|
||||
async with self.semaphore:
|
||||
async with self.client_session.head(url, timeout=5) as head:
|
||||
try:
|
||||
data = await head.read()
|
||||
_ = await head.read()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
if 'text/html' in head.headers['Content-Type']:
|
||||
@@ -130,7 +130,7 @@ class AsyncCrawler(object):
|
||||
print('Crawling: {0}'.format(self.baseurl))
|
||||
while len(to_crawl) > 0:
|
||||
discovered_urls = await self.run(urls=to_crawl)
|
||||
# empty toe crawl list and then add all newly discovered URLs for
|
||||
# empty to_crawl list and then add all newly discovered URLs for
|
||||
# the next iteration.
|
||||
to_crawl.clear()
|
||||
to_crawl.extend(discovered_urls)
|
||||
|
||||
Reference in New Issue
Block a user