package org.codelibs.riverweb.crawler;

import java.util.Date;
import java.util.Map;

import org.codelibs.fess.crawler.CrawlerThread;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.EsClient;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.riverweb.RiverWeb;
import org.codelibs.riverweb.config.RiverConfig;
import org.codelibs.riverweb.config.RiverConfigManager;
import org.codelibs.riverweb.util.ConversionUtil;
import org.codelibs.riverweb.util.ScriptUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.SortOrder;
import org.lastaflute.di.core.SingletonLaContainer;
import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class RwCrawlerThread extends CrawlerThread {
    private static final Logger logger = LoggerFactory.getLogger(RwCrawlerThread.class);

    @Override
    protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {
        final RiverConfigManager riverConfigManager = SingletonLaContainer.getComponent(RiverConfigManager.class);
        final RiverConfig riverConfig = riverConfigManager.get(crawlerContext.getSessionId());
        if (riverConfig.isIncremental()) {
            final EsClient esClient = SingletonLaContainer.getComponent(EsClient.class);
            try {
                final SearchResponse response = esClient.prepareSearch(riverConfig.getIndex()).setTypes(riverConfig.getType())
                        .setQuery(QueryBuilders.termQuery("url", urlQueue.getUrl())).addField("lastModified")
                        .addSort("lastModified", SortOrder.DESC).execute().actionGet();
                final SearchHits hits = response.getHits();
                if (hits.getTotalHits() > 0) {
                    final SearchHitField lastModifiedField = hits.getAt(0).getFields().get("lastModified");
                    if (lastModifiedField != null) {
                        final Date lastModified = ConversionUtil.convert(lastModifiedField.getValue(), Date.class);
                        if (lastModified != null) {
                            urlQueue.setLastModified(lastModified.getTime());
                        }
                    }
                }
            } catch (final Exception e) {
                logger.debug("Failed to retrieve lastModified.", e);
            }
        }
        return super.isContentUpdated(client, urlQueue);
    }

    protected boolean isValid(final UrlQueue<?> urlQueue) {
        if (urlQueue == null) {
            return false;
        }
        final RiverConfigManager riverConfigManager = SingletonLaContainer.getComponent(RiverConfigManager.class);
        final RiverConfig riverConfig = riverConfigManager.get(urlQueue.getSessionId());
        if (riverConfig != null) {
            final Map<String, Object> scriptSettings = riverConfig.getScriptSettings();
            if (scriptSettings != null) {
                Object result = ScriptUtils.execute(scriptSettings, "url_validation", v -> {
                    v.put("container", SingletonLaContainerFactory.getContainer());
                    v.put("urlQueue", urlQueue);
                    v.put("logger", RiverWeb.logger);
                });
                if (result instanceof Boolean) {
                    return ((Boolean) result).booleanValue();
                }
            }
        }
        return super.isValid(urlQueue);
    }
}

@@ -59,30 +59,8 @@
 ate;
-%0Aimport java.util.Map;
 %0A%0Aim
@@ -264,47 +264,8 @@
 ue;%0A
-import org.codelibs.riverweb.RiverWeb;%0A
 impo
@@ -419,55 +419,8 @@
 il;%0A
-import org.codelibs.riverweb.util.ScriptUtils;%0A
 impo
@@ -718,75 +718,8 @@
 er;%0A
-import org.lastaflute.di.core.factory.SingletonLaContainerFactory;%0A
 impo
@@ -2418,32 +2418,32 @@
     %7D%0A        %7D%0A
+
         return s
@@ -2492,1004 +2492,7 @@
    %7D
-%0A%0A    protected boolean isValid(final UrlQueue%3C?%3E urlQueue) %7B%0A        if (urlQueue == null) %7B%0A            return false;%0A        %7D%0A        final RiverConfigManager riverConfigManager = SingletonLaContainer.getComponent(RiverConfigManager.class);%0A        final RiverConfig riverConfig = riverConfigManager.get(urlQueue.getSessionId());%0A        if (riverConfig != null) %7B%0A            final Map%3CString, Object%3E scriptSettings = riverConfig.getScriptSettings();%0A            if (scriptSettings != null) %7B%0A                Object result = ScriptUtils.execute(scriptSettings, %22url_validation%22, v -%3E %7B%0A                    v.put(%22container%22, SingletonLaContainerFactory.getContainer());%0A                    v.put(%22urlQueue%22, urlQueue);%0A                    v.put(%22logger%22, RiverWeb.logger);%0A                %7D);%0A                if (result instanceof Boolean) %7B%0A                    return ((Boolean) result).booleanValue();%0A                %7D%0A            %7D%0A        %7D%0A        return super.isValid(urlQueue);%0A    %7D
 %0A%7D%0A

remove url_validation