/******************************************************
 * Web crawler
 * 
 * 
 * Copyright (C) 2013 by Peter Hedenskog (http://peterhedenskog.com)
 * 
 ****************************************************** 
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 * 
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 * 
 ******************************************************* 
 */
package com.soulgalore.crawler.core.assets.impl;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import org.jsoup.nodes.Document;

import com.google.inject.Inject;
import com.soulgalore.crawler.core.CrawlerConfiguration;
import com.soulgalore.crawler.core.HTMLPageResponse;
import com.soulgalore.crawler.core.assets.AssetFetcher;
import com.soulgalore.crawler.core.assets.AssetResponse;
import com.soulgalore.crawler.core.assets.AssetResponseCallable;
import com.soulgalore.crawler.core.assets.AssetURL;
import com.soulgalore.crawler.core.assets.AssetsParser;
import com.soulgalore.crawler.core.assets.AssetsVerificationResult;
import com.soulgalore.crawler.core.assets.AssetsVerifier;
import com.soulgalore.crawler.util.StatusCode;

public class DefaultAssetsVerifier implements AssetsVerifier {

  private final ExecutorService service;
  private final AssetFetcher responseCodeGetter;
  private final AssetsParser parser;

  @Inject
  public DefaultAssetsVerifier(ExecutorService theService, AssetFetcher getter,
      AssetsParser theParser) {
    service = theService;
    responseCodeGetter = getter;
    parser = theParser;

  }

  @Override
  public AssetsVerificationResult verify(Set<HTMLPageResponse> responses,
      CrawlerConfiguration configuration) {

    final Map<String, String> requestHeaders = configuration.getRequestHeadersMap();

    Set<AssetURL> urls = new HashSet<AssetURL>();

    final Set<Future<Set<AssetURL>>> fut = new HashSet<Future<Set<AssetURL>>>();

    for (HTMLPageResponse response : responses) {
      fut.add(service.submit(new AssetsParserCallable(response.getBody(), parser, response.getUrl())));
    }

    for (Future<Set<AssetURL>> future : fut) {
      try {
        urls.addAll(future.get());
      } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (ExecutionException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }

    Set<AssetResponse> working = new HashSet<AssetResponse>();
    Set<AssetResponse> nonWorking = new HashSet<AssetResponse>();

    final Map<Future<AssetResponse>, String> futures =
        new HashMap<Future<AssetResponse>, String>(urls.size());

    for (AssetURL url : urls) {
      futures.put(
          service.submit(new AssetResponseCallable(url.getURL(), responseCodeGetter, requestHeaders, url.getReferer())), url.getURL());

    }

    final Iterator<Entry<Future<AssetResponse>, String>> it = futures.entrySet().iterator();

    while (it.hasNext()) {

      final Entry<Future<AssetResponse>, String> entry = it.next();

      try {
        AssetResponse assetResponse = entry.getKey().get();
        if (StatusCode.isResponseCodeOk(assetResponse.getResponseCode()))
          working.add(assetResponse);
        else
          nonWorking.add(assetResponse);

      } catch (InterruptedException e) {
        // TODO fix real referer
        nonWorking.add(new AssetResponse(entry.getValue(),"INteruptedExpetion", StatusCode.SC_SERVER_RESPONSE_UNKNOWN
            .getCode(), -1));
      } catch (ExecutionException e) {
        nonWorking.add(new AssetResponse(entry.getValue(),"ExecutionException" + e.getCause(), StatusCode.SC_SERVER_RESPONSE_UNKNOWN
            .getCode(), -1));
      }

    }

    return new AssetsVerificationResult(working, nonWorking);
  }

  @Override
  public void shutdown() {
    service.shutdown();

  }

  private static class AssetsParserCallable implements Callable<Set<AssetURL>> {

    private final Document doc;
    private final AssetsParser parser;
    private final String referer;

    private AssetsParserCallable(Document theDoc, AssetsParser theParsers, String theReferer) {
      doc = theDoc;
      parser = theParsers;
      referer = theReferer;
    }

    @Override
    public Set<AssetURL> call() throws Exception {
      return parser.getAssets(doc, referer);
    }

  }
}

@@ -3997,41 +3997,8 @@
 ) %7B%0A
-        // TODO fix real referer%0A
     
@@ -4056,26 +4056,8 @@
 (),%22
-INteruptedExpetion
 %22, S
@@ -4154,32 +4154,74 @@
 nException e) %7B%0A
+        System.err.println(e.getCause());%0A
         nonWorki
@@ -4267,42 +4267,9 @@
 (),%22
-ExecutionException%22 + e.getCause()
+%22
 , St

remove outprint