"""
Load data from json file to ElasticSearch
Read an array of json objects, indexing each json object
"""

__author__ = 'saggu'

import json
from elasticsearch import Elasticsearch
from sys import stderr
import sys
import argparse

def loadDatainES(filename, index, doctype,dataFileType,hostname="localhost",port=9200,mappingFilePath=None):
    try:
        print "Connecting to " + hostname + " at port:" + str(port) 
        es = Elasticsearch([{'host': hostname, 'port': port}])
        
        if mappingFilePath:
            with open(mappingFilePath) as m:
                mapping = m.read()
                #print "Mapping file:" + mapping
                es.indices.create(index=index,  body=mapping,ignore=400)
                
        if dataFileType=="1":
            with open(filename) as f:   
                d = json.load(f)
                for wp in d:
                  res = es.index(index=index,doc_type=doctype,body=wp,id=wp["uri"])
                  print "indexing id: " + res["_id"] + " for uri: " + wp["uri"]
        elif dataFileType == "0":
            with open(filename) as f:
                lines = f.readlines()

                for line in lines:
                    if line.strip() != "":
                        jsonurlobj = json.loads(line.strip())
                        objkey = jsonurlobj['uri']
                        res = es.index(index=index,doc_type=doctype,body=line)
                        print "indexing id: " + res["_id"] + " for uri: " + objkey
                 
    except Exception, e:
        print >> stderr.write('ERROR: %s\n' % str(e))
        



if __name__ == '__main__':
    
    argp = argparse.ArgumentParser()
    argp.add_argument("-hostname",help="Elastic Search Server hostname, defaults to 'localhost'",default="localhost")
    argp.add_argument("-port",type=int, help="Elastic Search Server port,defaults to 9200",default=9200)
    argp.add_argument("-mappingFilePath", help="mapping/setting file for the index")
    argp.add_argument("filepath",help="json file to be loaded in ElasticSearch")
    argp.add_argument("indexname",help="desired name of the index in ElasticSearch")
    argp.add_argument("doctype",help="type of the document to be indexed")
    argp.add_argument("dataFileType", help="Specify '0' if every line in the data file is different json object  or '1' otherwise")
    arguments = argp.parse_args()
    
    if arguments.hostname and arguments.port:
        loadDatainES(arguments.filepath, arguments.indexname, arguments.doctype,arguments.dataFileType,arguments.hostname, arguments.port,arguments.mappingFilePath)
    else:
        loadDatainES(arguments.filepath, arguments.indexname, arguments.doctype,arguments.dataFileType,mappingFilePath)
    
    print "Thats all folks!"

@@ -476,16 +476,126 @@
 port%7D%5D)%0A
+         es = Elasticsearch(%5B'https://memex:%3Cpassword%3E@'+hostname + %22:%22 + str(port)%5D,show_ssl_warnings=False)%0A
         
@@ -1603,16 +1603,758 @@
  objkey%0A
+        elif dataFileType==%222%22:%0A            reader = SequenceFile.Reader(filename)%0A            key_class = reader.getKeyClass()%0A            value_class = reader.getValueClass()%0A%0A            key = key_class()%0A            value = value_class()%0A            %0A            position = reader.getPosition()%0A            %0A            while reader.next(key, value):%0A                if value.strip() != %22%22:%0A                    jsonurlobj = json.loads(value.strip())%0A                    objkey = jsonurlobj%5B'uri'%5D%0A                    res = es.index(index=index,doc_type=doctype,body=line,id=objkey)%0A                    print %22indexing id: %22 + res%5B%22_id%22%5D + %22 for uri: %22 + objkey%0A                position = reader.getPosition()%0A%0A            reader.close()%0A%0A
         

read sequencefile