/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.nutch.crawl;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * FetchSchedule which allows to reset the fetch interval to the default value
 */
public class CCFetchSchedule extends DefaultFetchSchedule {

  private final static Logger LOG = LoggerFactory
      .getLogger(CCFetchSchedule.class);

  private static final String RESET_FETCH_INTERVAL = "db.fetch.interval.reset";
  private static final String FETCH_TIME_MAX_DAYS_AHEAD = "db.fetch.time.max.days.ahead";

  private boolean resetFetchInterval = false;
  private boolean resetFetchTime = false;
  private long latestFetchTime;

  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null)
      return;
    resetFetchInterval = conf.getBoolean(RESET_FETCH_INTERVAL, false);
    if (resetFetchInterval) {
      resetFetchInterval = true;
      LOG.info("Resetting fetch interval if exceeding {} = {}",
          "db.fetch.interval.max", maxInterval);
    }
    latestFetchTime = System.currentTimeMillis();
    int fetchTimeMaxDaysAhead = conf.getInt(FETCH_TIME_MAX_DAYS_AHEAD, 0);
    if (fetchTimeMaxDaysAhead > 0) {
      resetFetchTime = true;
      latestFetchTime += fetchTimeMaxDaysAhead * 24 * 60 * 60 * 1000;
      LOG.info("Resetting fetch time if more than {} = {} days ahead",
          FETCH_TIME_MAX_DAYS_AHEAD, fetchTimeMaxDaysAhead);
    }
  }

  @Override
  public boolean shouldFetch(Text url, CrawlDatum datum, long curTime) {
    if (datum.getFetchTime() > curTime) {
      return false; // not time yet
    }
    return true;
  }

  @Override
  public CrawlDatum setPageGoneSchedule(Text url, CrawlDatum datum,
      long prevFetchTime, long prevModifiedTime, long fetchTime) {
    if (resetFetchInterval && datum.getFetchInterval() > maxInterval) {
      datum.setFetchInterval(maxInterval);
    }
    return super.setPageGoneSchedule(url, datum, prevFetchTime,
        prevModifiedTime, fetchTime);
  }

  @Override
  public CrawlDatum setPageRetrySchedule(Text url, CrawlDatum datum,
      long prevFetchTime, long prevModifiedTime, long fetchTime) {
    if (resetFetchInterval && datum.getFetchInterval() > maxInterval) {
      datum.setFetchInterval(maxInterval);
    }
    return super.setPageRetrySchedule(url, datum, prevFetchTime,
        prevModifiedTime, fetchTime);
  }

  @Override
  public CrawlDatum setFetchSchedule(Text url, CrawlDatum datum,
          long prevFetchTime, long prevModifiedTime,
          long fetchTime, long modifiedTime, int state) {
    if (resetFetchInterval && datum.getFetchInterval() > maxInterval) {
      datum.setFetchInterval(maxInterval);
    }
    if (resetFetchTime && datum.getFetchTime() > latestFetchTime) {
      datum.setFetchTime(latestFetchTime);
    }
    return super.setFetchSchedule(url, datum, prevFetchTime, prevModifiedTime,
        fetchTime, modifiedTime, state);
  }
}

@@ -1376,16 +1376,103 @@
 .ahead%22;
+%0A  private static final String RESET_NOT_MODIFIED = %22db.fetch.reset.notmodified.after%22;
 %0A%0A  priv
@@ -1584,16 +1584,52 @@
 tchTime;
+%0A  private long minModifiedTime = 0;
 %0A%0A  publ
@@ -2385,173 +2385,689 @@
 %7D%0A  
-%7D%0A%0A  @Override%0A  public boolean shouldFetch(Text url, CrawlDatum datum, long curTime) %7B%0A    if (datum.getFetchTime() %3E curTime) %7B%0A      return false; // not time yet
+  int secondsResetNotModifiedTime = conf.getInt(RESET_NOT_MODIFIED, 0);%0A    if (secondsResetNotModifiedTime %3E 0) %7B%0A      minModifiedTime = System.currentTimeMillis() - secondsResetNotModifiedTime * 1000;%0A    %7D%0A  %7D%0A%0A  @Override%0A  public boolean shouldFetch(Text url, CrawlDatum datum, long curTime) %7B%0A    if (datum.getFetchTime() %3E curTime) %7B%0A      return false; // not time yet%0A    %7D%0A    if (datum.getModifiedTime() %3E 0%0A        && datum.getStatus() == CrawlDatum.STATUS_DB_NOTMODIFIED%0A        && datum.getModifiedTime() %3C minModifiedTime) %7B%0A      // trigger a full re-fetch of not-modified pages%0A      // (do not send if-not-modified-since requests)%0A      datum.setModifiedTime(0);
 %0A   

reset not-modified time during fetch list generation if the time elapsed since the last real fetch exceeds a configurable interval (db.fetch.reset.notmodified.after). This avoids that pages are archived as revisit records for an overlong time span.