package org.kagan.core;

import java.io.IOException;
import java.util.Date;
import java.util.concurrent.TimeUnit;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.kagan.config.Configure;

public final class HtmlParser {
    
    private final IdleConnectionMonitorThread monitor;
    private final CloseableHttpClient httpClient;
    private static final HtmlParser me = new HtmlParser();
    private final PoolingHttpClientConnectionManager connMgr;
    private static final int CONNECT_TIMEOUT = 8000;
    
    private HtmlParser() {
        this.connMgr = new PoolingHttpClientConnectionManager();
        this.connMgr.setDefaultMaxPerRoute(10);
        this.connMgr.setMaxTotal(Configure.websiteNum * 50);
        this.httpClient = HttpClients.custom().setDefaultRequestConfig(
            RequestConfig.custom().setSocketTimeout(CONNECT_TIMEOUT).setConnectTimeout(CONNECT_TIMEOUT).build()
        ).setConnectionManager(this.connMgr).build();
        
        monitor = new IdleConnectionMonitorThread(this.connMgr);
        monitor.start();
    }
    
    public static Document parse(String url, String charset) {
        try {
            HttpGet httpGet = new HttpGet(url);
            httpGet.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 SE 2.X MetaSr 1.0");
            CloseableHttpResponse response = me.httpClient.execute(httpGet, HttpClientContext.create());
            try {
                int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode == HttpStatus.SC_OK) {
                    HttpEntity entity = response.getEntity();
                    return Jsoup.parse(EntityUtils.toString(entity, charset));
                } else {
                    System.out.println(String.format("[%s] - HtmlParser: %s - Status Code : %d.", new Date(), url, statusCode));
                }
            } finally {
                response.close();
            }
        } catch (Exception e) {
            System.out.println(String.format("[%s] - HtmlParser: %s %s.", new Date(), url, e.getMessage()));
        }
        
        return null;
    }
    
    public static final void shutdown() {
        try {
            me.monitor.shutdown();
            me.httpClient.close();
            me.connMgr.shutdown();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    public static class IdleConnectionMonitorThread extends Thread {
        
        private volatile boolean shutdown;
        private final PoolingHttpClientConnectionManager connMgr;
        private static final int WAIT_TIMEOUT = 8000;
        
        public IdleConnectionMonitorThread(PoolingHttpClientConnectionManager connMgr) {
            this.connMgr = connMgr;
        }

        @Override
        public void run() {
            try {
                while (!shutdown) {
                    synchronized (this) {
                        wait(WAIT_TIMEOUT);
                        connMgr.closeExpiredConnections();
                        connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
                    }
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        
        public void shutdown() {
            shutdown = true;
            synchronized(this) {
                notifyAll();
            }
        }
        
    }
}

@@ -206,22 +206,137 @@
 ttp.Http
-Status
+Request;%0Aimport org.apache.http.HttpResponse;%0Aimport org.apache.http.HttpStatus;%0Aimport org.apache.http.ProtocolException
 ;%0Aimport
@@ -599,24 +599,84 @@
 HttpClient;%0A
+import org.apache.http.impl.client.DefaultRedirectStrategy;%0A
 import org.a
@@ -780,16 +780,61 @@
 anager;%0A
+import org.apache.http.protocol.HttpContext;%0A
 import o
@@ -1673,16 +1673,60 @@
         
+).setRedirectStrategy(new RedirectStrategy()
 ).setCon
@@ -3226,32 +3226,32 @@
 ntStackTrace();%0A
-
         %7D%0A    %7D%0A
@@ -3247,32 +3247,784 @@
    %7D%0A    %7D%0A    %0A
+    public static class RedirectStrategy extends DefaultRedirectStrategy %7B%0A%0A        @Override%0A        public boolean isRedirected(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException %7B%0A            boolean isRedirect = false;%0A            try %7B%0A                isRedirect = super.isRedirected(request, response, context);%0A            %7D catch (ProtocolException e) %7B%0A                e.printStackTrace();%0A            %7D%0A            if (!isRedirect) %7B%0A                int responseCode = response.getStatusLine().getStatusCode();%0A                if (responseCode == 301 %7C%7C responseCode == 302) %7B%0A                    return true;%0A                %7D%0A            %7D%0A            return isRedirect;%0A        %7D%0A        %0A    %7D%0A    %0A
     public stati

Add redirect Strategy