import os
import os.path
import hashlib
import errno
from tqdm import tqdm


def gen_bar_updator(pbar):
    def bar_update(count, block_size, total_size):
        pbar.total = total_size / block_size
        pbar.update(count)

    return bar_update


def check_integrity(fpath, md5):
    if not os.path.isfile(fpath):
        return False
    md5o = hashlib.md5()
    with open(fpath, 'rb') as f:
        # read in 1MB chunks
        for chunk in iter(lambda: f.read(1024 * 1024), b''):
            md5o.update(chunk)
    md5c = md5o.hexdigest()
    if md5c != md5:
        return False
    return True


def download_url(url, root, filename, md5):
    from six.moves import urllib

    root = os.path.expanduser(root)
    fpath = os.path.join(root, filename)

    try:
        os.makedirs(root)
    except OSError as e:
        if e.errno == errno.EEXIST:
            pass
        else:
            raise

    # downloads file
    if os.path.isfile(fpath) and check_integrity(fpath, md5):
        print('Using downloaded and verified file: ' + fpath)
    else:
        try:
            print('Downloading ' + url + ' to ' + fpath)
            urllib.request.urlretrieve(url, fpath, reporthook=gen_bar_updator(tqdm()))
        except:
            if url[:5] == 'https':
                url = url.replace('https:', 'http:')
                print('Failed download. Trying https -> http instead.'
                      ' Downloading ' + url + ' to ' + fpath)
                urllib.request.urlretrieve(url, fpath)


def list_dir(root, prefix=False):
    """List all directories at a given root

    Args:
        root (str): Path to directory whose folders need to be listed
        prefix (bool, optional): If true, prepends the path to each result, otherwise
            only returns the name of the directories found
    """
    root = os.path.expanduser(root)
    directories = list(
        filter(
            lambda p: os.path.isdir(os.path.join(root, p)),
            os.listdir(root)
        )
    )

    if prefix is True:
        directories = [os.path.join(root, d) for d in directories]

    return directories


def list_files(root, suffix, prefix=False):
    """List all files ending with a suffix at a given root

    Args:
        root (str): Path to directory whose folders need to be listed
        suffix (str or tuple): Suffix of the files to match, e.g. '.png' or ('.jpg', '.png').
            It uses the Python "str.endswith" method and is passed directly
        prefix (bool, optional): If true, prepends the path to each result, otherwise
            only returns the name of the files found
    """
    root = os.path.expanduser(root)
    files = list(
        filter(
            lambda p: os.path.isfile(os.path.join(root, p)) and p.endswith(suffix),
            os.listdir(root)
        )
    )

    if prefix is True:
        files = [os.path.join(root, d) for d in files]

    return files

@@ -87,17 +87,17 @@
 ar_updat
-o
+e
 r(pbar):
@@ -160,33 +160,114 @@
     
-pbar.total = total_size /
+if pbar.total is None and total_size:%0A            pbar.total = total_size%0A        progress_bytes = count *
  blo
@@ -294,21 +294,39 @@
 .update(
-count
+progress_bytes - pbar.n
 )%0A%0A    r
@@ -1268,19 +1268,52 @@
 eve(
-url, fpath,
+%0A                url, fpath,%0A               
  rep
@@ -1337,18 +1337,56 @@
 pdat
-o
+e
 r(tqdm(
-))
+unit='B', unit_scale=True))%0A            
 )%0A  
@@ -1663,26 +1663,145 @@
 etrieve(
-url, fpath
+%0A                    url, fpath,%0A                    reporthook=gen_bar_updater(tqdm(unit='B', unit_scale=True))%0A                
 )%0A%0A%0Adef 

Fix broken progress bar (#524)