r00t2/optools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

quick gzip fix

Browse Source
This commit is contained in:
brent s 2017-11-26 09:00:47 -05:00
parent 045bcfaa0d
commit 2d42adaaf7
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
net/mirroring/check.py
View File

@ -57,7 +57,15 @@ class website(object):


def remoteFetch(self): def remoteFetch(self):
with urlopen(self.meta['url']) as _site: with urlopen(self.meta['url']) as _site:
self.site['remote'] = _site.read().decode('utf-8') self.site['remote'] = _site.read()
self.headers = dict(_site.info())
# Handle gzip encoding
if 'Content-Encoding' in self.headers.keys():
if self.headers['Content-Encoding'] == 'gzip':
from gzip import decompress
self.site['remote'] = decompress(self.site['remote']).decode('utf-8')
else:
self.site['remote'] = self.site['remote'].decode('utf-8')
_hash = hashlib.sha256(self.site['remote'].encode('utf-8')) _hash = hashlib.sha256(self.site['remote'].encode('utf-8'))
self.site['remotesum'] = str(_hash.hexdigest()) self.site['remotesum'] = str(_hash.hexdigest())
self.meta['timestamp'] = str(int(datetime.datetime.now().timestamp())) self.meta['timestamp'] = str(int(datetime.datetime.now().timestamp()))