/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package at.illecker.storm.commons.tokenizer;

import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import at.illecker.storm.commons.Configuration;
import at.illecker.storm.commons.Dataset;
import at.illecker.storm.commons.tweet.TokenizedTweet;
import at.illecker.storm.commons.tweet.Tweet;
import at.illecker.storm.commons.util.HtmlUtils;
import at.illecker.storm.commons.util.RegexUtils;
import at.illecker.storm.commons.util.UnicodeUtils;
import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.ling.Word;
import edu.stanford.nlp.process.PTBTokenizer.PTBTokenizerFactory;
import edu.stanford.nlp.process.TokenizerFactory;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;

public class Tokenizer {
  private static final Logger LOG = LoggerFactory.getLogger(Tokenizer.class);

  public static List<String> tokenize(String str) {
    // Step 1) Trim text
    str = str.trim();

    // Step 2) Replace Unicode symbols \u0000
    if (UnicodeUtils.containsUnicode(str)) {
      String replacedText = UnicodeUtils.replaceUnicodeSymbols(str);
      // LOG.info("Replaced Unicode symbols from '" + str + "' to '"
      // + replacedText + "'");
      if (replacedText.equals(str)) {
        LOG.error("Unicode symbols could not be replaced: '" + str + "'");
      }
      str = replacedText;
    }

    // Step 3) Replace HTML symbols &#[0-9];
    if (HtmlUtils.containsHtml(str)) {
      String replacedText = HtmlUtils.replaceHtmlSymbols(str);
      // LOG.info("Replaced HTML symbols from '" + text + "' to '"
      // + replacedText + "'");
      if (replacedText.equals(str)) {
        LOG.error("HTML symbols could not be replaced: '" + str + "'");
      }
      str = replacedText;
    }

    // Step 4) Tokenize string by multiple regex patterns
    List<String> resultTokens = new ArrayList<String>();
    Matcher m = RegexUtils.TOKENIZER_PATTERN.matcher(str);
    while (m.find()) {
      resultTokens.add(m.group());
    }
    return resultTokens;
  }

  public static List<TokenizedTweet> tokenizeTweets(List<Tweet> tweets) {
    List<TokenizedTweet> tokenizedTweets = new ArrayList<TokenizedTweet>();
    for (Tweet tweet : tweets) {
      List<List<String>> tokenizedSentences = new ArrayList<List<String>>();
      tokenizedSentences.add(tokenize(tweet.getText()));
      tokenizedTweets.add(new TokenizedTweet(tweet.getId(), tweet.getText(),
          tweet.getScore(), tokenizedSentences));
    }
    return tokenizedTweets;
  }

  public static List<List<HasWord>> tokenizeTreebank(String str) {
    TokenizerFactory<Word> tokenizer = PTBTokenizerFactory
        .newTokenizerFactory();
    tokenizer.setOptions("ptb3Escaping=false");

    return MaxentTagger.tokenizeText(new StringReader(str), tokenizer);
  }

  public static void main(String[] args) {
    boolean extendedTest = true;
    List<Tweet> tweets = null;

    // load tweets
    if (extendedTest) {
      Dataset dataset = Configuration.getDataSetSemEval2013();
      tweets = dataset.getTrainTweets(true);
    } else { // test tweets
      tweets = Tweet.getTestTweets();
      tweets
          .add(new Tweet(
              0L,
              ":-))) xDD XDD ;) :) :-) :) :D :o) :] :3 :c) :> =] 8) =) :} :^) :-D 8-D 8D x-D xD X-D XD =-D =D =-3 =3"));
      tweets
          .add(new Tweet(0L,
              ">:[ :-( :(  :-c :c :-< :< :-[ :[ :{ ;( :-|| :@ >:( :'-( :'( :'-) :') \\m/"));
      tweets
          .add(new Tweet(
              0L,
              "\"All the money you've spent on my credit card I'm taking it out of your account\".... Hi I'm Sydney and I'm filing bankruptcy\ud83d\ude05\ud83d\ude05\ud83d\ude05\ud83d\ude05 \ud83d\ude05"));
      tweets
          .add(new Tweet(
              0L,
              "word:-) Moon:Oct Jobs! Thursday:... http://t.co/TZZzrrKa +1 (000) 123-4567, (000) 123-4567, and 123-4567 "));
      tweets
          .add(new Tweet(
              0L,
              "t/m k/o w/my b/slisten Rt/follow S/o S/O O/U O/A w/ w/Biden w/deals w/you w/the w/her"));
      tweets
          .add(new Tweet(0L, "5pm 5Am 5% $5 5-6am 7am-10pm U.S. U.K. L.O.V.E"));
    }

    // Tokenize tweets
    long startTime = System.currentTimeMillis();
    for (Tweet tweet : tweets) {
      List<String> tokens = Tokenizer.tokenize(tweet.getText());
      LOG.info("Tweet: '" + tweet + "'");
      LOG.info("Tokens: " + tokens);
    }
    LOG.info("Tokenize finished after "
        + (System.currentTimeMillis() - startTime) + " ms");
  }
}

@@ -1107,63 +1107,8 @@
 et;%0A
-import at.illecker.storm.commons.tweet.TokenizedTweet;%0A
 impo
@@ -2057,21 +2057,20 @@
     LOG.
-error
+warn
 (%22Unicod
@@ -2460,13 +2460,12 @@
 LOG.
-error
+warn
 (%22HT
@@ -2849,84 +2849,20 @@
 ist%3C
-TokenizedTweet%3E tokenizeTweets(List%3CTweet%3E tweets) %7B%0A    List%3CTokenizedTweet
+List%3CString%3E
 %3E to
@@ -2871,73 +2871,26 @@
 nize
-d
 Tweets
- = new ArrayList%3CTokenizedTweet%3E();%0A    for (Tweet tweet :
+(List%3CTweet%3E
  twe
@@ -2888,34 +2888,32 @@
 weet%3E tweets) %7B%0A
-  
     List%3CList%3CSt
@@ -2928,24 +2928,21 @@
 okenized
-Sentence
+Tweet
 s = new 
@@ -2976,60 +2976,36 @@
     
-  tokenizedSentences.add(tokenize(tweet.getText()));
+for (Tweet tweet : tweets) %7B
 %0A   
@@ -3031,105 +3031,32 @@
 add(
-new T
+t
 okenize
-dTweet(tweet.getId(), tweet.getText(),%0A          tweet.getScore(), tokenizedSentences
+(tweet.getText()
 ));%0A
@@ -5085,10 +5085,11 @@
 %22);%0A  %7D%0A
+%0A
 %7D%0A

Fix Tokenizer