/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package org.apache.geode.management.internal.configuration;

import java.util.Properties;
import java.util.concurrent.TimeUnit;

import org.awaitility.Awaitility;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import org.apache.geode.distributed.ConfigurationProperties;
import org.apache.geode.distributed.internal.InternalDistributedSystem;
import org.apache.geode.distributed.internal.InternalLocator;
import org.apache.geode.distributed.internal.membership.gms.MembershipManagerHelper;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.IgnoredException;
import org.apache.geode.test.dunit.rules.ClusterStartupRule;
import org.apache.geode.test.dunit.rules.MemberVM;
import org.apache.geode.test.junit.categories.DistributedTest;
import org.apache.geode.test.junit.rules.GfshCommandRule;

@Category(DistributedTest.class)
public class ClusterConfigLocatorRestartDUnitTest {

  @Rule
  public ClusterStartupRule rule = new ClusterStartupRule(5);

  @Rule
  public GfshCommandRule gfsh = new GfshCommandRule();

  @Test
  public void serverRestartsAfterLocatorReconnects() throws Exception {
    Properties props = new Properties();
    props.setProperty(ConfigurationProperties.MAX_WAIT_TIME_RECONNECT, "5000");
    MemberVM locator0 = rule.startLocatorVM(0, props);

    rule.startServerVM(1, props, locator0.getPort());
    MemberVM server2 = rule.startServerVM(2, props, locator0.getPort());

    server2.invokeAsync(() -> MembershipManagerHelper
        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance()));
    locator0.invokeAsync(() -> MembershipManagerHelper
        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance()));

    waitForLocatorToReconnect(locator0);

    rule.startServerVM(3, locator0.getPort());

    gfsh.connectAndVerify(locator0);
    gfsh.executeAndAssertThat("list members").statusIsSuccess().tableHasColumnOnlyWithValues("Name",
        "locator-0", "server-1", "server-2", "server-3");
  }

  @Test
  public void serverRestartsAfterOneLocatorDies() throws Exception {
    IgnoredException.addIgnoredException("This member is no longer in the membership view");
    IgnoredException.addIgnoredException("This node is no longer in the membership view");

    // Otherwise we get a graceful shutdown...
    Host.getHost(0).getVM(0).invoke(() -> {
      if (InternalDistributedSystem.shutdownHook != null) {
        Runtime.getRuntime().removeShutdownHook(InternalDistributedSystem.shutdownHook);
      }
    });

    Properties props = new Properties();
    props.setProperty(ConfigurationProperties.MAX_WAIT_TIME_RECONNECT, "5000");
    MemberVM locator0 = rule.startLocatorVM(0, props);
    MemberVM locator1 = rule.startLocatorVM(1, props, locator0.getPort());

    MemberVM server2 = rule.startServerVM(2, props, locator0.getPort(), locator1.getPort());
    MemberVM server3 = rule.startServerVM(3, props, locator0.getPort(), locator1.getPort());

    // Go away and don't come back
    locator0.invokeAsync(() -> System.exit(1));

    server3.invokeAsync(() -> MembershipManagerHelper
        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance()));

    waitForLocatorToReconnect(locator1);

    rule.startServerVM(4, locator1.getPort(), locator0.getPort());

    gfsh.connectAndVerify(locator1);
    gfsh.executeAndAssertThat("list members").statusIsSuccess().tableHasColumnOnlyWithValues("Name",
        "locator-1", "server-2", "server-3", "server-4");

    // Recover the VM so that subsequent rule cleanup works
    Host.getHost(0).getVM(0).bounce();
  }

  private void waitForLocatorToReconnect(MemberVM locator) {
    Awaitility.waitAtMost(30, TimeUnit.SECONDS).until(() -> locator.invoke(() -> {
      InternalLocator intLocator = InternalLocator.getLocator();
      return intLocator != null && intLocator.isSharedConfigurationRunning();
    }));
  }
}

@@ -1419,16 +1419,55 @@
 eption;%0A
+import org.apache.geode.test.dunit.VM;%0A
 import o
@@ -2703,19 +2703,89 @@
 ator0);%0A
+%0A
    
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)%0A        .until(() -%3E
  gfsh.ex
@@ -2828,32 +2828,45 @@
 tatusIsSuccess()
+%0A            
 .tableHasColumnO
@@ -2882,32 +2882,24 @@
 lues(%22Name%22,
-%0A       
  %22locator-0%22
@@ -2935,16 +2935,17 @@
 rver-3%22)
+)
 ;%0A  %7D%0A%0A 
@@ -3912,35 +3912,22 @@
  // 
-Go away and don't come back
+Shut down hard
 %0A   
@@ -3968,24 +3968,116 @@
 .exit(1));%0A%0A
+    // Recover the VM so that subsequent rule cleanup works%0A    locator0.getVM().bounce();%0A%0A
     server3.
@@ -4350,19 +4350,89 @@
 ator1);%0A
+%0A
    
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)%0A        .until(() -%3E
  gfsh.ex
@@ -4483,16 +4483,29 @@
 uccess()
+%0A            
 .tableHa
@@ -4533,24 +4533,16 @@
 (%22Name%22,
-%0A       
  %22locato
@@ -4586,107 +4586,8 @@
 -4%22)
-;%0A%0A    // Recover the VM so that subsequent rule cleanup works%0A    Host.getHost(0).getVM(0).bounce(
 );%0A 

Fix ClusterConfigLocatorRestartDUnitTest intermittent failures (#1664)