
import os
from urlparse import urlparse
import zipfile 

import click

import adapters
import utils
from filters import BasicFilterer

@click.command()
@click.argument('sources', type=click.Path(exists=True), required=True)
@click.argument('output', type=click.Path(exists=True), required=True)
@click.option('--force', is_flag=True)
def process(sources, output, force):
    """Download sources and process the file to the output directory.

    \b
    SOURCES: Source JSON file or directory of files. Required.
    OUTPUT: Destination directory for generated data. Required.
    """
    catalog_features = []
    for path in utils.get_files(sources):
        pathparts = utils.get_path_parts(path)
        pathparts[0] = output.strip(os.sep)
        pathparts[-1] = pathparts[-1].replace('.json', '.geojson')

        outdir = os.sep.join(pathparts[:-1])
        outfile = os.sep.join(pathparts)

        source = utils.read_json(path)
        urlfile = urlparse(source['url']).path.split('/')[-1]

        if not hasattr(adapters, source['filetype']):
            utils.error('Unknown filetype', source['filetype'], '\n')
            continue

        if os.path.isfile(outfile) and not force:
            utils.error('Skipping', path, 'since generated file exists.',
                        'Use --force to regenerate.', '\n')
            continue

        utils.info('Downloading', source['url'])

        try:
            fp = utils.download(source['url'])
        except IOError:
            utils.error('Failed to download', source['url'], '\n')
            continue

        utils.info('Reading', urlfile)

        if 'filter' in source:
            filterer = BasicFilterer(source['filter'], source.get('filter_operator', 'and'))
        else:
            filterer = None

        try:
            geojson = getattr(adapters, source['filetype'])\
                .read(fp, source['properties'],
                    filterer=filterer,
                    source_filename=source.get("filenameInZip", None))
        except IOError:
            utils.error('Failed to read', urlfile)
            continue
        except zipfile.BadZipfile, e:
            utils.error('Unable to open zip file', source['url'])
            continue
        finally:
            os.remove(fp.name)

        excluded_keys = ['filetype', 'url', 'properties', 'filter', 'filenameInZip']
        properties = {k:v for k,v in source.iteritems() if k not in excluded_keys}
        properties['source_url'] = source['url']

        geojson['properties'] = properties

        utils.make_sure_path_exists(outdir)
        utils.write_json(outfile, geojson)

        properties['path'] = outfile
        catalog_entry = {
            'type': 'Feature',
            'properties': properties,
            'geometry': {
                'type': 'Polygon',
                'coordinates': utils.polygon_from_bbox(geojson['bbox'])
            }
        }
        catalog_features.append(catalog_entry)

        utils.success('Done. Processed to', outfile, '\n')

    catalog = {
        'type': 'FeatureCollection',
        'features': catalog_features
    }
    utils.write_json(os.path.join(output,'catalog.geojson'), catalog)

if __name__ == '__main__':
    process()

@@ -49,16 +49,28 @@
 zipfile 
+%0Aimport json
 %0A%0Aimport
@@ -1348,34 +1348,158 @@
         
-continue%0A%0A
+with open(outfile, %22rb%22) as f:%0A                geojson = json.load(f)%0A            properties = geojson%5B'properties'%5D%0A        else:%0A   
         
 utils.in
@@ -1482,32 +1482,33 @@
 lse:%0A           
+ 
 utils.info('Down
@@ -1533,16 +1533,20 @@
 url'%5D)%0A%0A
+    
         
@@ -1562,16 +1562,20 @@
         
+    
 fp = uti
@@ -1601,32 +1601,36 @@
 'url'%5D)%0A        
+ 
+   
 except IOError:%0A
@@ -1621,32 +1621,36 @@
 except IOError:%0A
+    
             util
@@ -1704,32 +1704,36 @@
 n')%0A            
+   
+ 
 continue%0A%0A      
@@ -1718,32 +1718,36 @@
       continue%0A%0A
+    
         utils.in
@@ -1766,24 +1766,28 @@
 , urlfile)%0A%0A
+    
         if '
@@ -1809,32 +1809,36 @@
 ce:%0A            
+    
 filterer = Basic
@@ -1902,38 +1902,46 @@
 'and'))%0A        
+    
 else:%0A
+    
             filt
@@ -1953,37 +1953,45 @@
 = None%0A%0A        
+    
 try:%0A
+    
             geoj
@@ -2051,16 +2051,20 @@
         
+    
 .read(fp
@@ -2099,32 +2099,36 @@
                 
+    
 filterer=filtere
@@ -2146,24 +2146,28 @@
             
+  
+  
 source_filen
@@ -2205,32 +2205,36 @@
  None))%0A        
+    
 except IOError:%0A
@@ -2237,32 +2237,36 @@
 or:%0A            
+   
+ 
 utils.error('Fai
@@ -2288,16 +2288,20 @@
 rlfile)%0A
+    
         
@@ -2317,24 +2317,28 @@
 nue%0A        
+    
 except zipfi
@@ -2359,32 +2359,36 @@
  e:%0A            
+ 
+   
 utils.error('Una
@@ -2429,32 +2429,36 @@
 '%5D)%0A            
+    
 continue%0A       
@@ -2458,16 +2458,20 @@
         
+   
+ 
 finally:
@@ -2471,16 +2471,20 @@
 inally:%0A
+    
         
@@ -2503,24 +2503,28 @@
 e(fp.name)%0A%0A
+    
         excl
@@ -2596,32 +2596,36 @@
 InZip'%5D%0A        
+ 
+   
 properties = %7Bk:
@@ -2683,32 +2683,36 @@
 d_keys%7D%0A        
+    
 properties%5B'sour
@@ -2741,24 +2741,28 @@
 '%5D%0A%0A        
+  
+  
 geojson%5B'pro
@@ -2789,24 +2789,28 @@
 es%0A%0A        
+   
+ 
 utils.make_s
@@ -2833,16 +2833,20 @@
 outdir)%0A
+    
         
@@ -2879,16 +2879,84 @@
 geojson)
+%0A    %0A            utils.success('Done. Processed to', outfile, '%5Cn')
 %0A%0A      
@@ -3290,67 +3290,8 @@
 y)%0A%0A
-        utils.success('Done. Processed to', outfile, '%5Cn')%0A
 %0A   

Include metadata for existing files that are not being regenerated in output