misc/web-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improved content-type detection

Browse Source
This commit is contained in:
Simon Weald
2018-09-06 17:08:12 +01:00
parent e57a86c60a
commit 6d9103c154
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
utils/helpers.py
View File

@@ -62,7 +62,7 @@ class WebPage(object):
request = urllib.request.Request(self.url, headers=self.headers) request = urllib.request.Request(self.url, headers=self.headers)
page = urllib.request.urlopen(request, timeout=5) page = urllib.request.urlopen(request, timeout=5)
headers = page.info() headers = page.info()
if headers['content-type'] == "text/html": if "text/html" in headers['content-type']:
self.source = page.read() self.source = page.read()