@@ -0,0 +1,2315 @@
+import requests%0Aimport json%0Aimport sys%0Aimport os%0A%0A%0Adef grab_source(url, output):%0A    %22%22%22%0A    Grab a source from a url and store it in an output file%0A%0A    This creates uses requests as a dependency because I'm lazy.%0A    It probably would have taken me less time to just write it with%0A    urllib than writing this docstring%0A    %22%22%22%0A%0A    # We use stream because these files might be huge%0A    response = requests.get(url, stream=True)%0A%0A    # We don't do anything if there's something wrong with the url%0A    # This is basically what made urllib.urlretrieve a hassle%0A    if not response.ok:%0A        return%0A%0A    with open(output, 'w') as output_file:%0A        for block in response.iter_content(1024):%0A            output_file.write(block)%0A%0A%0Adef archive(directory):%0A    %22%22%22%0A    Archive a OpenSpending dataset export directory%0A    %22%22%22%0A%0A    # If we accidentally pass in something that's not a directory%0A    # we don't do anything%0A    if not os.path.isdir(directory):%0A        return%0A%0A    # Check if the directory contains a dataset.json file%0A    dataset = os.path.join(directory, 'dataset.json')%0A    if not os.path.isfile(dataset):%0A        return%0A%0A    # Open the dataset.json file and grab the sources listed in it%0A    with open(dataset) as descriptor:%0A        data = json.load(descriptor)%0A        if len(data%5B'sources'%5D):%0A            # Create an archive directory because there are some%0A            # sources we want to grab%0A            archive_directory = os.path.join(directory, 'archive')%0A            if not os.path.exists(archive_directory):%0A                os.makedirs(archive_directory)            %0A%0A            # Loop through sources, grab them and store in an output file%0A            # called %3Csource_id%3E.csv%0A            for source in data%5B'sources'%5D:%0A                filename = '%7B0%7D.csv'.format(source%5B'id'%5D)%0A                archive_file = os.path.join(archive_directory, filename)%0A                grab_source(source%5B'url'%5D, output=archive_file)%0A%0A            # If the archive directory is empty which will happen if%0A            # grabbing the sources failed for some reason%0A            if not os.listdir(archive_directory):%0A                os.rmdir(archive_directory)%0A%0A%0Aif __name__ == %22__main__%22:%0A    # Loop through each of the arguments and archive them%0A    for directory in sys.argv%5B1:%5D:%0A        archive(directory)%0A

Add source archiving to OpenSpending export (separate script)