/*
 * Licensed to ElasticSearch and Shay Banon under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.cloud.aws.blobstore;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.blobstore.ImmutableBlobContainer;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;

import java.util.ArrayList;
import java.util.concurrent.Executor;

/**
 *
 */
public class S3BlobStore extends AbstractComponent implements BlobStore {

    private final AmazonS3 client;

    private final String bucket;

    private final String region;

    private final Executor executor;

    private final int bufferSizeInBytes;

    public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, Executor executor) {
        super(settings);
        this.client = client;
        this.bucket = bucket;
        this.region = region;
        this.executor = executor;

        this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();

        if (!client.doesBucketExist(bucket)) {
            if (region != null) {
                client.createBucket(bucket, region);
            } else {
                client.createBucket(bucket);
            }
        }
    }

    @Override
    public String toString() {
        return (region == null ? "" : region + "/") + bucket;
    }

    public AmazonS3 client() {
        return client;
    }

    public String bucket() {
        return bucket;
    }

    public Executor executor() {
        return executor;
    }

    public int bufferSizeInBytes() {
        return bufferSizeInBytes;
    }

    @Override
    public ImmutableBlobContainer immutableBlobContainer(BlobPath path) {
        return new S3ImmutableBlobContainer(path, this);
    }

    @Override
    public void delete(BlobPath path) {
        ObjectListing prevListing = null;
        while (true) {
            ObjectListing list;
            if (prevListing != null) {
                list = client.listNextBatchOfObjects(prevListing);
            } else {
                list = client.listObjects(bucket, path.buildAsString("/") + "/");
            }
            //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
            //we can do at most 1K objects per delete
            int objectCount = 0;
            DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
            ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
            for (S3ObjectSummary summary : list.getObjectSummaries()) {
                objectCount++;
                keys.add(new KeyVersion(summary.getKey()));
                //Every 500 objects batch the delete request
                if (objectCount > 500) {
                    multiObjectDeleteRequest.setKeys(keys);
                    client.deleteObjects(multiObjectDeleteRequest);
                    multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                    keys.clear();
                    objectCount = 0;
                }
            }
            if (objectCount > 0) {
                multiObjectDeleteRequest.setKeys(keys);
                client.deleteObjects(multiObjectDeleteRequest);
            }
            if (list.isTruncated()) {
                prevListing = list;
            } else {
                break;
            }
        }
    }

    @Override
    public void close() {
    }
}

@@ -3155,16 +3155,397 @@
 = null;%0A
+        //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html%0A        //we can do at most 1K objects per delete%0A        int objectCount = 0;%0A        //We don't know the bucket name until first object listing%0A        DeleteObjectsRequest multiObjectDeleteRequest = null;%0A        ArrayList%3CKeyVersion%3E keys = new ArrayList%3CKeyVersion%3E();%0A
         
@@ -3816,241 +3816,12 @@
     
-%7D%0A
     
-        //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html%0A            //we can do at most 1K objects per delete%0A            int objectCount = 0;%0A            DeleteObjectsRequest 
 mult
@@ -3907,65 +3907,9 @@
     
-ArrayList%3CKeyVersion%3E keys = new ArrayList%3CKeyVersion%3E();
+%7D
 %0A   
@@ -4516,23 +4516,26 @@
 if (
-objectCount %3E 0
+list.isTruncated()
 ) %7B%0A
@@ -4554,163 +4554,71 @@
     
-multiObjectDeleteRequest.setKeys(keys);%0A                client.deleteObjects(multiObjectDeleteRequest);%0A            %7D%0A            if (list.isTruncated()) %7B
+prevListing = list;%0A            %7D else %7B%0A                break;
 %0A   
@@ -4630,89 +4630,162 @@
     
+%7D%0A
     
-prevListing = list;%0A            %7D else %7B%0A                break;%0A            %7D
+    %7D%0A        if (objectCount %3E 0) %7B%0A            multiObjectDeleteRequest.setKeys(keys);%0A            client.deleteObjects(multiObjectDeleteRequest);
 %0A   

Improve S3 grouped deletes