package org.apache.solr.cloud;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.util.ArrayList;
import java.util.List;

import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.SolrCore;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.DirectUpdateHandler2;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;

@Slow
@BadApple
public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
  
  private static final int BASE_RUN_LENGTH = 120000;
  private static final int RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", Integer.toString(BASE_RUN_LENGTH)));

  @BeforeClass
  public static void beforeSuperClass() {

  }
  
  @AfterClass
  public static void afterSuperClass() {
    
  }
  
  @Before
  @Override
  public void setUp() throws Exception {
    useFactory("solr.StandardDirectoryFactory");

    super.setUp();
    
    System.setProperty("numShards", Integer.toString(sliceCount));
  }
  
  @Override
  @After
  public void tearDown() throws Exception {
    System.clearProperty("numShards");
    super.tearDown();
    resetExceptionIgnores();
  }
  
  public ChaosMonkeySafeLeaderTest() {
    super();
    sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "3"));
    shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "12"));
  }
  
  @Override
  public void doTest() throws Exception {
    
    handle.clear();
    handle.put("QTime", SKIPVAL);
    handle.put("timestamp", SKIPVAL);
    
    // randomly turn on 5 seconds 'soft' commit
    randomlyEnableAutoSoftCommit();

    del("*:*");
    
    List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
    int threadCount = 2;
    for (int i = 0; i < threadCount; i++) {
      StopableIndexingThread indexThread = new StopableIndexingThread(10000 + i*50000, true);
      threads.add(indexThread);
      indexThread.start();
    }
    
    chaosMonkey.startTheMonkey(false, 500);
    int runLength = RUN_LENGTH;
    Thread.sleep(runLength);
    
    chaosMonkey.stopTheMonkey();
    
    for (StopableIndexingThread indexThread : threads) {
      indexThread.safeStop();
    }
    
    // wait for stop...
    for (StopableIndexingThread indexThread : threads) {
      indexThread.join();
    }
    
    for (StopableIndexingThread indexThread : threads) {
      assertEquals(0, indexThread.getFails());
    }
    
    // try and wait for any replications and what not to finish...

    Thread.sleep(2000);

    waitForThingsToLevelOut(180000);

    checkShardConsistency(true, true);
    
    if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
  }

  private void randomlyEnableAutoSoftCommit() {
    if (r.nextBoolean()) {
      log.info("Turning on auto soft commit");
      for (CloudJettyRunner jetty : shardToJetty.get("shard1")) {
        SolrCore core = ((SolrDispatchFilter) jetty.jetty.getDispatchFilter()
            .getFilter()).getCores().getCore("collection1");
        try {
          ((DirectUpdateHandler2) core.getUpdateHandler()).getCommitTracker()
              .setTimeUpperBound(5000);
        } finally {
          core.close();
        }
      }
    } else {
      log.info("Not turning on auto soft commit");
    }
  }
  
  // skip the randoms - they can deadlock...
  @Override
  protected void indexr(Object... fields) throws Exception {
    SolrInputDocument doc = new SolrInputDocument();
    addFields(doc, fields);
    addFields(doc, "rnd_b", true);
    indexDoc(doc);
  }

}

@@ -1082,16 +1082,57 @@
 cument;%0A
+import org.apache.solr.core.Diagnostics;%0A
 import o
@@ -1157,24 +1157,24 @@
 e.SolrCore;%0A
-
 import org.a
@@ -1264,16 +1264,66 @@
 ndler2;%0A
+import org.apache.solr.update.SolrCmdDistributor;%0A
 import o
@@ -1433,18 +1433,8 @@
 low%0A
-@BadApple%0A
 publ
@@ -1768,17 +1768,467 @@
 ass() %7B%0A
+    SolrCmdDistributor.testing_errorHook = new Diagnostics.Callable() %7B%0A      @Override%0A      public void call(Object... data) %7B%0A        SolrCmdDistributor.Request sreq = (SolrCmdDistributor.Request)data%5B1%5D;%0A        if (sreq.exception == null) return;%0A        if (sreq.exception.getMessage().contains(%22Timeout%22)) %7B%0A          Diagnostics.logThreadDumps(%22REQUESTING THREAD DUMP DUE TO TIMEOUT: %22 + sreq.exception.getMessage());%0A        %7D%0A      %7D%0A    %7D;
 %0A
-
   %7D%0A  %0A 
@@ -2281,20 +2281,64 @@
 ass() %7B%0A
-
     
+SolrCmdDistributor.testing_errorHook = null;
 %0A  %7D%0A  %0A

enable chaos safe leader test