/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.spi.impl.operationservice.impl;

import com.hazelcast.config.Config;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.MemberLeftException;
import com.hazelcast.instance.MemberImpl;
import com.hazelcast.internal.cluster.impl.MembershipUpdateTest.StaticMemberNodeContext;
import com.hazelcast.spi.Operation;
import com.hazelcast.spi.properties.GroupProperty;
import com.hazelcast.test.HazelcastParallelClassRunner;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.TestHazelcastInstanceFactory;
import com.hazelcast.test.annotation.ParallelTest;
import com.hazelcast.test.annotation.QuickTest;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import static com.hazelcast.instance.HazelcastInstanceFactory.newHazelcastInstance;
import static org.junit.Assert.fail;

@RunWith(HazelcastParallelClassRunner.class)
@Category({QuickTest.class, ParallelTest.class})
public class Invocation_OnMemberLeftTest extends HazelcastTestSupport {

    private OperationServiceImpl localOperationService;
    private InvocationMonitor localInvocationMonitor;
    private HazelcastInstance remote;
    private MemberImpl remoteMember;
    private TestHazelcastInstanceFactory instanceFactory;

    @Before
    public void setup() {
        instanceFactory = createHazelcastInstanceFactory();
        Config config = new Config();
        config.setProperty(GroupProperty.MAX_JOIN_SECONDS.getName(), "5");

        HazelcastInstance[] cluster = instanceFactory.newInstances(config, 2);

        localOperationService = getOperationServiceImpl(cluster[0]);
        localInvocationMonitor = localOperationService.getInvocationMonitor();

        remote = cluster[1];
        remoteMember = (MemberImpl) remote.getCluster().getLocalMember();
    }

    @Test
    public void whenMemberLeaves() throws Exception {
        Future<Object> future =
                localOperationService.invokeOnTarget(null, new UnresponsiveTargetOperation(), remoteMember.getAddress());

        remote.getLifecycleService().terminate();

        try {
            future.get();
            fail("Invocation should have failed with MemberLeftException!");
        } catch (MemberLeftException e) {
            ignore(e);
        }
    }

    @Test
    public void whenMemberRestarts_withSameAddress() throws Exception {
       whenMemberRestarts(new Runnable() {
           @Override
           public void run() {
               remote = instanceFactory.newHazelcastInstance(remoteMember.getAddress());
           }
       });
    }

    @Test
    public void whenMemberRestarts_withSameIdentity() throws Exception {
        whenMemberRestarts(new Runnable() {
            @Override
            public void run() {
                StaticMemberNodeContext nodeContext = new StaticMemberNodeContext(instanceFactory, remoteMember);
                remote = newHazelcastInstance(new Config(), remoteMember.toString(), nodeContext);
            }
        });
    }

    private void whenMemberRestarts(Runnable restartAction) throws Exception {
        Future<Object> futureBeforeShutdown =
                localOperationService.invokeOnTarget(null, new UnresponsiveTargetOperation(), remoteMember.getAddress());

        final CountDownLatch blockMonitorLatch = new CountDownLatch(1);
        final CountDownLatch resumeMonitorLatch = new CountDownLatch(1);

        localInvocationMonitor.execute(new Runnable() {
            @Override
            public void run() {
                blockMonitorLatch.countDown();
                assertOpenEventually(resumeMonitorLatch);
            }
        });
        assertOpenEventually(blockMonitorLatch);

        remote.getLifecycleService().terminate();
        restartAction.run();

        Future<Object> futureAfterRestart =
                localOperationService.invokeOnTarget(null, new UnresponsiveTargetOperation(), remoteMember.getAddress());

        resumeMonitorLatch.countDown();

        try {
            futureBeforeShutdown.get();
            fail("Invocation should have failed with MemberLeftException!");
        } catch (MemberLeftException e) {
            ignore(e);
        }

        try {
            futureAfterRestart.get(1, TimeUnit.SECONDS);
            fail("future.get() should have failed with TimeoutException!");
        } catch (TimeoutException e) {
            ignore(e);
        }
    }

    private static class UnresponsiveTargetOperation extends Operation {
        @Override
        public void run() throws Exception {
        }

        @Override
        public boolean returnsResponse() {
            return false;
        }
    }
}

@@ -1021,16 +1021,54 @@
 operty;%0A
+import com.hazelcast.test.AssertTask;%0A
 import c
@@ -1696,24 +1696,113 @@
 stInstance;%0A
+import static org.junit.Assert.assertNotNull;%0Aimport static org.junit.Assert.assertTrue;%0A
 import stati
@@ -4663,24 +4663,358 @@
 torLatch);%0A%0A
+        // Unresponsive operation should be executed before shutting down the node%0A        assertTrueEventually(new AssertTask() %7B%0A            @Override%0A            public void run() throws Exception %7B%0A                assertNotNull(remote.getUserContext().get(UnresponsiveTargetOperation.COMPLETION_FLAG));%0A            %7D%0A        %7D);%0A%0A
         remo
@@ -5077,24 +5077,87 @@
 ion.run();%0A%0A
+        assertTrue(remote.getLifecycleService().isRunning());%0A%0A
         Futu
@@ -5870,62 +5870,258 @@
     
-@Override%0A        public void run() throws Exception %7B
+static final String COMPLETION_FLAG = UnresponsiveTargetOperation.class.getName();%0A%0A        @Override%0A        public void run() throws Exception %7B%0A            getNodeEngine().getHazelcastInstance().getUserContext().put(COMPLETION_FLAG, new Object());
 %0A   

Fix Invocation_OnMemberLeft member restart tests