import json
import requests
import argparse
import dataset

def fetch_builds_data(jenkins_url):
    '''Get json data of all Jenkins builds

    JENKINS_URL/api/json?
      tree=jobs[name,builds[number,result,duration,builtOn,id,timestamp,fullDisplayName]]

    Return builds_data dictionary with following schema::

     [
      {
        "name" : <job_name>,
        "builds" : [
                     {
                       "duration" : <build_duration>,
                       "timestamp" : <build_timestamp>,
                       "id" : <build_id>,
                       "url" : <build_url>,
                       "number" : <build_number>,
                       "result" : <build_result>,
                       "builtOn" : <node>,
                       "fullDisplayName": <build_display_name>,
                     }
                   ],
                   ...
      },
      ...

     ]
    '''

    params = {}
    params['tree'] = 'jobs[name,builds[number,url,result,duration,builtOn,id,timestamp,fullDisplayName]]'
    r = requests.get(
        "%s/api/json" % jenkins_url,
        verify=False,
        params=params
    )
    builds_data = json.loads(r.text)["jobs"]

    return builds_data

def store_builds_data(builds_data, dbname):
    '''Saves builds_data in SQLite database

    Database has one table `builds` with following columns::

       index           = 5
       builtOn         = slave01.example.com
       name            = check_shell
       timestamp       = 1432739306340
       number          = 113
       id              = 2015-05-27_15-08-26
       result          = SUCCESS
       duration        = 3796
       fullDisplayName = check_shell #113

    '''

    db = dataset.connect('sqlite:///%s' % dbname)

    table = db.get_table('builds', primary_id='index')

    db.begin()
    for job_entry in builds_data:
        for build in job_entry['builds']:

            # FIXME: Build DB is append-only, thus, for efficiency, we
            # should process only new builds with timestamp later than
            # the last one which alredy exists in db

            build['name'] = job_entry['name']
            table.upsert(build, ['name','number'])
    db.commit()

    return len(db['builds'])

def update_builds_db(dbname, source_file=None, source_url=None):

    if source_file:
        with open(source_file, 'r') as f:
            builds_data = json.loads(f.readline())['jobs']
    elif source_url:
        builds_data = fetch_builds_data(source_url)
    else:
        raise ValueError("No URL and no source file specified")

    return store_builds_data(builds_data, dbname)

def update_db(dbname, source):

    source_file = source.get('file')
    source_url = source.get('url')

    if source_file:
        with open(source_file, 'r') as f:
            builds_data = json.loads(f.readline())['jobs']
    elif source_url:
        builds_data = fetch_builds_data(source_url)
    else:
        raise ValueError("No URL and no source file specified")

    return store_builds_data(builds_data, dbname)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--datafile',
                        help="file with JSON source data",
                        default=None,
    )
    parser.add_argument('--url',
                        help="URL of Jenkins instance",
                        default=None,
    )
    parser.add_argument('-d', '--dbname',
                        help="Filename of the SQLite database",
                        default='output.db'
    )


    args = parser.parse_args()

    update_builds_db(dbname=args.dbname, source_file=args.datafile, source_url=args.url)

@@ -44,23 +44,76 @@
 %0Aimport 
-dataset
+logging%0Aimport dataset%0A%0Alogger = logging.getLogger(__name__)
 %0A%0Adef fe
@@ -1233,16 +1233,71 @@
 %22jobs%22%5D%0A
+    logger.info(%22Fetched %25d builds%22 %25 len(builds_data))
 %0A    ret
@@ -3179,16 +3179,98 @@
 in__':%0A%0A
+    logger.setLevel(logging.INFO)%0A    logger.addHandler(logging.StreamHandler())%0A%0A
     pars
@@ -3741,38 +3741,269 @@
  )%0A%0A
-%0A    args = parser.parse_args(
+    parser.add_argument('-v', '--verbose',%0A                        help=%22Enable debug output%22,%0A                        action=%22store_true%22,%0A    )%0A%0A    args = parser.parse_args()%0A%0A    if args.verbose:%0A        logger.setLevel(logging.DEBUG)%0A%0A    logger.debug(args
 )%0A%0A 

Add logging to fetch_data script