package org.meaningfulweb.api;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.tika.config.TikaConfig;
import org.apache.tika.mime.MimeType;
import org.apache.tika.mime.MimeTypes;
import org.meaningfulweb.cext.Extract;
import org.meaningfulweb.cext.HtmlContentProcessorFactory;
import org.meaningfulweb.cext.HtmlExtractor;
import org.meaningfulweb.util.http.HttpComponentsServiceImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ExtractorCLI {

  private static Logger LOG = LoggerFactory.getLogger(ExtractorCLI.class);

  public static void main(String[] args)
    throws Exception {

    // the config file and the url
    File configFile = new File("config/cextr.json");
    String url = "http://twitpic.com/3sryl9";
    HtmlContentProcessorFactory processorFactory = new HtmlContentProcessorFactory(
      configFile);
    HtmlExtractor htmlExtractor = new HtmlExtractor();
    htmlExtractor.setProcessorFactory(processorFactory);
    HttpComponentsServiceImpl httpclient = new HttpComponentsServiceImpl();
    httpclient.initialize();

    // create base config
    Map<String, Object> config = new HashMap<String, Object>();
    config.put("perComponentDOM", false);
    config.put("perPipelineDOM", true);
    config.put("fullcontent.extractHtml", true);
    config.put("fullcontent.extractText", true);

    // create base metadata
    Map<String, Object> metadata = new HashMap<String, Object>();
    metadata.put("url", url);

    // create the pipelines and components to run
    List<String> pipelines = new ArrayList<String>();
    List<String> components = new ArrayList<String>();
    components.add("fullcontent");

    // get the html from the url
    long start = System.currentTimeMillis();
    byte[] contentBytes = null;
    try {
      contentBytes = httpclient.get(url);
    }
    catch (IOException e) {
      LOG.debug("Error getting url info: " + url, e);
    }

    if (contentBytes != null && contentBytes.length > 0) {

      MimeTypes detector = TikaConfig.getDefaultConfig().getMimeRepository();
      MimeType mimeType = detector.getMimeType(contentBytes);
      if (mimeType.getName().contains("html")) {

        Extract extract = new Extract();
        extract.setConfig(config);
        extract.setContent(contentBytes);
        extract.setMetadata(metadata);

        try {

          // add extraction pipelines
          if (pipelines != null && pipelines.size() > 0) {
            extract.getPipelines().addAll(pipelines);
          }

          // add extraction components
          if (components != null && components.size() > 0) {
            extract.getComponents().addAll(components);
          }

          htmlExtractor.extract(extract);
        }
        catch (Exception e) {
          LOG.error("Error extracting content", e);
        }

        // translate from the extract name to field names
        System.out.println(extract.getExtracted());

      }
    }

    httpclient.shutdown();
  }
}

@@ -818,16 +818,18 @@
 %22);%0A    
+//
 String u
@@ -862,16 +862,97 @@
 sryl9%22;%0A
+    String url = %22http://www.youtube.com/watch?v=7rE0-ek6MZA&feature=topvideos%22;%0A
     Html
@@ -1444,32 +1444,217 @@
     
-config.put(%22fullcontent.
+%0A    %0A    Map elementConfig = new HashMap();    %0A    List%3CString%3E elementsList = new ArrayList%3CString%3E();%0A    elementsList.add(%22title%22);%0A    elementConfig.put(%22elements%22, elementsList);%0A    elementConfig.put(%22
 extr
@@ -1663,19 +1663,20 @@
 tHtml%22, 
-tru
+fals
 e);%0A    
@@ -1691,38 +1691,169 @@
 ut(%22
-fullcontent.extractText%22, true
+element%22, elementConfig);%0A    %0A    Map opengraphConfig = new HashMap(); %0A    opengraphConfig.put(%22includeAll%22, true);%0A    config.put(%22opengraph%22, opengraphConfig
 );%0A%0A
@@ -2140,16 +2140,18 @@
 ();%0A    
+//
 componen
@@ -2172,16 +2172,223 @@
 ntent%22);
+%0A    //components.add(%22bestimage%22);%0A    components.add(%22opengraph%22);%0A    components.add(%22element%22);%0A    //components.add(%22article%22);%0A    %0A    //pipelines.add(%22metadata%22);%0A    //pipelines.add(%22content%22);%0A    
 %0A%0A    //

clean up