@@ -0,0 +1,965 @@
+from requests import get%0Afrom shutil import copyfileobj%0Afrom hashlib import sha256%0Aimport gzip%0A%0A# fetch and save to file%0Adef fetch_and_save(url, name):%0A%09file = get(url, stream=True)%0A%09with open(name, 'wb') as out:%0A%09%09copyfileobj(file.raw, out)%0A%0A# generate sha265 of file%0Adef hashfile(file):%0A%09with open(file, %22rb%22) as infile:%0A%09%09f = infile.read()%0A%09hash = sha256()%0A%09hash.update(f)%0A%09return hash.hexdigest()%0A%0A# generate sha265 of stream directly from url (to use for modification checks)%0Adef fetch_and_hash(url):%0A%09file = get(url, stream=True)%0A%09f = file.raw.read()%0A%09hash = sha256()%0A%09hash.update(f)%0A%09return hash.hexdigest()%0A%0A# some files show up gzizpped, and this saves text files in binary. The function below gets them out either way.%0Adef open_binary_string_file(filename):%0A%09try:%0A%09%09with open(filename, %22rb%22) as f:%0A%09%09%09bytes = f.read()%0A%09%09%09return bytes.decode()%0A%09except UnicodeDecodeError:%0A%09%09with gzip.open(filename, %22rb%22) as f:%0A%09%09%09bytes = f.read()%0A%09%09%09return bytes.decode()%0A

Create hash.py