/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.master;

import alluxio.Configuration;
import alluxio.PropertyKey;
import alluxio.master.PrimarySelector.State;
import alluxio.master.journal.JournalSystem;
import alluxio.master.journal.JournalSystem.Mode;
import alluxio.util.CommonUtils;
import alluxio.util.ThreadUtils;

import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

import javax.annotation.concurrent.NotThreadSafe;

/**
 * The fault tolerant version of {@link AlluxioMaster} that uses zookeeper and standby masters.
 */
@NotThreadSafe
final class FaultTolerantAlluxioMasterProcess extends AlluxioMasterProcess {
  private static final Logger LOG =
      LoggerFactory.getLogger(FaultTolerantAlluxioMasterProcess.class);

  private final long mServingThreadTimeoutMs =
      Configuration.getMs(PropertyKey.MASTER_SERVING_THREAD_TIMEOUT);

  private PrimarySelector mLeaderSelector;
  private Thread mServingThread;

  /**
   * Creates a {@link FaultTolerantAlluxioMasterProcess}.
   */
  protected FaultTolerantAlluxioMasterProcess(JournalSystem journalSystem,
      PrimarySelector leaderSelector) {
    super(journalSystem);
    try {
      stopServing();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
    mLeaderSelector = Preconditions.checkNotNull(leaderSelector, "leaderSelector");
    mServingThread = null;
  }

  @Override
  public void start() throws Exception {
    mJournalSystem.start();
    try {
      mLeaderSelector.start(getRpcAddress());
    } catch (IOException e) {
      LOG.error(e.getMessage(), e);
      throw new RuntimeException(e);
    }

    while (!Thread.interrupted()) {
      if (mServingThread == null) {
        // We are in secondary mode. Nothing to do until we become the primary.
        mLeaderSelector.waitForState(State.PRIMARY);
        LOG.info("Transitioning from secondary to primary");
        mJournalSystem.setMode(Mode.PRIMARY);
        stopMasters();
        LOG.info("Secondary stopped");
        startMasters(true);
        mServingThread = new Thread(() -> {
          startServing(" (gained leadership)", " (lost leadership)");
        }, "MasterServingThread");
        mServingThread.start();
        LOG.info("Primary started");
      } else {
        // We are in primary mode. Nothing to do until we become the secondary.
        mLeaderSelector.waitForState(State.SECONDARY);
        LOG.info("Transitioning from primary to secondary");
        stopServing();
        mServingThread.join(mServingThreadTimeoutMs);
        if (mServingThread.isAlive()) {
          LOG.error(
              "Failed to stop serving thread after {}ms. Printing serving thread stack trace "
                  + "and exiting.\n{}",
              mServingThreadTimeoutMs, ThreadUtils.formatStackTrace(mServingThread));
          System.exit(-1);
        }
        mServingThread = null;
        stopMasters();
        mJournalSystem.setMode(Mode.SECONDARY);
        LOG.info("Primary stopped");
        startMasters(false);
        LOG.info("Secondary started");
      }
    }
  }

  @Override
  public void stop() throws Exception {
    super.stop();
    if (mLeaderSelector != null) {
      mLeaderSelector.stop();
    }
  }

  @Override
  public void waitForReady() {
    CommonUtils.waitFor(this + " to start", new Function<Void, Boolean>() {
      @Override
      public Boolean apply(Void input) {
        return (mServingThread == null || isServing());
      }
    });
  }
}

@@ -882,16 +882,73 @@
 itions;%0A
+import org.apache.commons.lang.exception.ExceptionUtils;%0A
 import o
@@ -2699,16 +2699,34 @@
 () -%3E %7B%0A
+          try %7B%0A  
         
@@ -2783,24 +2783,382 @@
 dership)%22);%0A
+          %7D catch (Throwable t) %7B%0A            Throwable root = ExceptionUtils.getRootCause(t);%0A            if ((root != null && (root instanceof InterruptedException)) %7C%7C Thread.interrupted()) %7B%0A              return;%0A            %7D%0A            LOG.error(%22Exception thrown in main serving thread. System exiting.%22, t);%0A            System.exit(-1);%0A          %7D%0A
         %7D, %22

Kill the server when the serving thread fails (#6879)