package de.jungblut.crawl.extraction;

import static de.jungblut.crawl.extraction.OutlinkExtractor.consumeStream;
import static de.jungblut.crawl.extraction.OutlinkExtractor.extractOutlinks;
import static de.jungblut.crawl.extraction.OutlinkExtractor.getConnection;

import java.io.IOException;
import java.io.InputStream;
import java.util.HashSet;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringEscapeUtils;
import org.htmlparser.util.ParserException;

import de.jungblut.crawl.ConsoleResultWriter;
import de.jungblut.crawl.Crawler;
import de.jungblut.crawl.FetchResult;
import de.jungblut.crawl.SequentialCrawler;
import de.jungblut.crawl.extraction.ArticleContentExtrator.ContentFetchResult;
import de.l3s.boilerpipe.BoilerpipeExtractor;
import de.l3s.boilerpipe.extractors.ArticleExtractor;

/**
 * Extractor for news articles. Uses Boilerpipes {@link ArticleExtractor} to
 * extract the largest block of text and the article title.
 * 
 * @author thomas.jungblut
 * 
 */
public final class ArticleContentExtrator implements
    Extractor<ContentFetchResult> {

  private final BoilerpipeExtractor extractor = ArticleExtractor.getInstance();

  // I know I catch hell for parsing HTML with REGEX
  private final Pattern titleExtractor = Pattern
      .compile("<title>(.*?)</title>");

  @Override
  public ContentFetchResult extract(String site) {

    if (site == null || !site.startsWith("http") || site.length() > 500)
      return null;

    try {
      InputStream connection = getConnection(site);
      String html = consumeStream(connection);
      html = StringEscapeUtils.unescapeHtml(html);
      final HashSet<String> outlinkSet = extractOutlinks(html, site);

      Matcher matcher = titleExtractor.matcher(html);
      String title = "";
      if (matcher.find()) {
        title = matcher.group(1);
      }
      String extractedLargestText = extractor.getText(html);
      return new ContentFetchResult(site, outlinkSet, title,
          extractedLargestText);
    } catch (ParserException pEx) {
      // ignore parser exceptions, they contain mostly garbage
    } catch (RuntimeException rEx) {
      rEx.printStackTrace();
    } catch (Exception e) {
      System.err.println(e.toString().replace("\n", "; ") + " >>> URL was: \""
          + site + "\"");
    }

    return null;
  }

  /**
   * Article content fetch result.
   */
  public static class ContentFetchResult extends FetchResult {

    private final String title;
    private final String text;

    public ContentFetchResult(String url, HashSet<String> outlinks) {
      super(url, outlinks);
      title = null;
      text = null;
    }

    public ContentFetchResult(String url, HashSet<String> outlinks,
        String title, String text) {
      super(url, outlinks);
      this.title = title;
      this.text = text;
    }

    public String getTitle() {
      return title;
    }

    public String getText() {
      return text;
    }

    @Override
    public String toString() {
      return title + "\n\n" + text;
    }

  }

  public static void main(String[] args) throws IOException,
      InterruptedException, ExecutionException {
    String start = "http://www.spiegel.de/wissenschaft/natur/erbgut-entziffert-austern-haben-viele-anti-stress-gene-a-856902.html";

    Crawler<ContentFetchResult> crawler = new SequentialCrawler<>(1,
        new ArticleContentExtrator(),
        new ConsoleResultWriter<ContentFetchResult>());

    crawler.process(start);

  }
}

@@ -394,165 +394,333 @@
 on;%0A
+%0A
 import 
-java.util.regex.Matcher;%0Aimport java.util.regex.Pattern;%0A%0Aimport org.apache.commons.lang.StringEscapeUtils;%0Aimport org.htmlparser.util.ParserException
+org.apache.commons.lang.StringEscapeUtils;%0Aimport org.htmlparser.NodeFilter;%0Aimport org.htmlparser.Parser;%0Aimport org.htmlparser.filters.NodeClassFilter;%0Aimport org.htmlparser.tags.TitleTag;%0Aimport org.htmlparser.util.NodeList;%0Aimport org.htmlparser.util.ParserException;%0Aimport org.htmlparser.util.SimpleNodeIterator
 ;%0A%0Ai
@@ -1417,145 +1417,96 @@
 %0A%0A  
-// I know I catch hell for parsing HTML with REGEX%0A  private final Pattern titleExtractor = Pattern
+private static final NodeFilter TITLE_FILTER = new NodeClassFilter(
 %0A      
-.compile(%22%3Ctitle%3E(.*?)%3C/title%3E%22
+TitleTag.class
 );%0A%0A
@@ -1896,157 +1896,49 @@
 e);%0A
-%0A      Matcher matcher = titleExtractor.matcher(html);%0A      String title = %22%22;%0A      if (matcher.find()) %7B%0A        title = matcher.group(1);%0A      %7D
+      String title = extractTitle(html);%0A
 %0A   
@@ -2416,16 +2416,542 @@
 l;%0A  %7D%0A%0A
+  /**%0A   * Extracts the title from the given HTML.%0A   * %0A   * @return never null, just an empty string if not parsable.%0A   */%0A  public static String extractTitle(String html) throws ParserException %7B%0A    String title = %22%22;%0A    Parser parser = new Parser(html);%0A    NodeList matches = parser.extractAllNodesThatMatch(TITLE_FILTER);%0A    SimpleNodeIterator it = matches.elements();%0A    while (it.hasMoreNodes()) %7B%0A      TitleTag node = (TitleTag) it.nextNode();%0A      title = node.getTitle().trim();%0A    %7D%0A    return title;%0A  %7D%0A%0A
   /**%0A  

stop parsing with regex