@@ -0,0 +1,2551 @@
+#!/usr/bin/python%0A'''Process G6 JSON files into elasticsearch%0A%0AThis version reads G6 JSON from disk or DM API.%0A%0AUsage:%0A    process-g6-into-elastic-search.py %3Ces_endpoint%3E %3Cdir_or_endpoint%3E %5B%3Ctoken%3E%5D%0A%0AArguments:%0A    es_endpoint      Full ES index URL%0A    dir_or_endpoint  Directory path to import or an API URL if token is given%0A    token            Digital Marketplace API token%0A%0A'''%0A%0Aimport os%0Aimport sys%0Aimport json%0Aimport urllib2%0A%0A%0Adef post_to_es(es_endpoint, json_data):%0A    handler = urllib2.HTTPHandler()%0A    opener = urllib2.build_opener(handler)%0A%0A    if not es_endpoint.endswith('/'):%0A        es_endpoint += '/'%0A    request = urllib2.Request(es_endpoint + str(json_data%5B'id'%5D),%0A                              data=json.dumps(json_data))%0A    request.add_header(%22Content-Type%22, 'application/json')%0A%0A    print request.get_full_url()%0A    # print request.get_data()%0A%0A    try:%0A        connection = opener.open(request)%0A    except urllib2.HTTPError, e:%0A        connection = e%0A        print connection%0A%0A    # check. Substitute with appropriate HTTP code.%0A    if connection.code == 200:%0A        data = connection.read()%0A        print str(connection.code) + %22 %22 + data%0A    else:%0A        print %22connection.code = %22 + str(connection.code)%0A%0A%0Adef request_services(endpoint, token):%0A    handler = urllib2.HTTPBasicAuthHandler()%0A    opener = urllib2.build_opener(handler)%0A%0A    page_url = endpoint%0A    while page_url:%0A        print %22requesting %7B%7D%22.format(page_url)%0A%0A        request = urllib2.Request(page_url)%0A        request.add_header(%22Authorization%22, %22Bearer %7B%7D%22.format(token))%0A        response = opener.open(request).read()%0A%0A        data = json.loads(response)%0A        for service in data%5B%22services%22%5D:%0A            yield service%0A%0A        page_url = filter(lambda l: l%5B'rel'%5D == 'next', data%5B'links'%5D)%0A        if page_url:%0A            page_url = page_url%5B0%5D%5B'href'%5D%0A%0A%0Adef process_json_files_in_directory(dirname):%0A    for filename in os.listdir(dirname):%0A        with open(os.path.join(dirname, filename)) as f:%0A            data = json.loads(f.read())%0A            print %22doing %22 + filename%0A            yield data%0A%0A%0Adef main():%0A    if len(sys.argv) == 4:%0A        es_endpoint, endpoint, token = sys.argv%5B1:%5D%0A        for data in request_services(endpoint, token):%0A            post_to_es(es_endpoint, data)%0A    elif len(sys.argv) == 3:%0A        es_endpoint, listing_dir = sys.argv%5B1:%5D%0A        for data in process_json_files_in_directory(listing_dir):%0A            post_to_es(es_endpoint, data)%0A    else:%0A        print __doc__%0A%0Aif __name__ == '__main__':%0A    main()%0A

Move ElasticSearch import script from buyer-frontend into search-api