/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package opennlp.tools.tokenize;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;

import opennlp.tools.util.StringUtil;

/**
 * Generate events for maxent decisions for tokenization.
 */
public class DefaultTokenContextGenerator implements TokenContextGenerator {
  
  private final Set<String> inducedAbbreviations;
  
  /**
   * Creates a default context generator for tokenizer.
   */
  public DefaultTokenContextGenerator() {
    this(Collections.<String>emptySet());
  }
  
  /**
   * Creates a default context generator for tokenizer.
   * 
   * @param inducedAbbreviations the induced abbreviations
   */
  public DefaultTokenContextGenerator(Set<String> inducedAbbreviations) {
    this.inducedAbbreviations = inducedAbbreviations;
  }

  /* (non-Javadoc)
   * @see opennlp.tools.tokenize.TokenContextGenerator#getContext(java.lang.String, int)
   */
  public String[] getContext(String sentence, int index) {
    List<String> preds = new ArrayList<String>();
    String prefix = sentence.substring(0, index);
    String suffix = sentence.substring(index);
    preds.add("p=" + prefix);
    preds.add("s=" + suffix);
    if (index > 0) {
      addCharPreds("p1", sentence.charAt(index - 1), preds);
      if (index > 1) {
        addCharPreds("p2", sentence.charAt(index - 2), preds);
        preds.add("p21=" + sentence.charAt(index - 2) + sentence.charAt(index - 1));
      }
      else {
        preds.add("p2=bok");
      }
      preds.add("p1f1=" + sentence.charAt(index - 1) + sentence.charAt(index));
    }
    else {
      preds.add("p1=bok");
    }
    addCharPreds("f1", sentence.charAt(index), preds);
    if (index+1 < sentence.length()) {
      addCharPreds("f2", sentence.charAt(index + 1), preds);
      preds.add("f12=" + sentence.charAt(index) + sentence.charAt(index + 1));
    }
    else {
      preds.add("f2=bok");
    }
    if (sentence.charAt(0) == '&' && sentence.charAt(sentence.length() - 1) == ';') {
      preds.add("cc");//character code
    }
    
    if(index == sentence.length() - 1 && inducedAbbreviations.contains(sentence)) {
      preds.add("pabb");
    }
    
    if(inducedAbbreviations.contains(sentence)) {
      preds.add("abb");
    }

    String[] context = new String[preds.size()];
    preds.toArray(context);
    return context;
  }


  /**
   * Helper function for getContext.
   */
  private void addCharPreds(String key, char c, List<String> preds) {
    preds.add(key + "=" + c);
    if (Character.isLetter(c)) {
      preds.add(key + "_alpha");
      if (Character.isUpperCase(c)) {
        preds.add(key + "_caps");
      }
    }
    else if (Character.isDigit(c)) {
      preds.add(key + "_num");
    }
    else if (StringUtil.isWhitespace(c)) {
      preds.add(key + "_ws");
    }
    else {
      if (c=='.' || c=='?' || c=='!') {
        preds.add(key + "_eos");
      }
      else if (c=='`' || c=='"' || c=='\'') {
        preds.add(key + "_quote");
      }
      else if (c=='[' || c=='{' || c=='(') {
        preds.add(key + "_lp");
      }
      else if (c==']' || c=='}' || c==')') {
        preds.add(key + "_rp");
      }
    }
  }
}

@@ -1122,21 +1122,23 @@
 %0A  %0A  pr
-ivate
+otected
  final S
@@ -1767,24 +1767,736 @@
 nt index) %7B%0A
+    List%3CString%3E preds = createContext(sentence, index);%0A    String%5B%5D context = new String%5Bpreds.size()%5D;%0A    preds.toArray(context);%0A    return context;%0A  %7D%0A%0A  /**%0A   * Returns an %7B@link ArrayList%7D of features for the specified sentence string%0A   * at the specified index. Extensions of this class can override this method%0A   * to create a customized %7B@link TokenContextGenerator%7D%0A   * %0A   * @param sentence%0A   *          the token been analyzed%0A   * @param index%0A   *          the index of the character been analyzed%0A   * @return an %7B@link ArrayList%7D of features for the specified sentence string%0A   *         at the specified index.%0A   */%0A  protected List%3CString%3E createContext(String sentence, int index) %7B%0A
     List%3CStr
@@ -3760,99 +3760,20 @@
     
-String%5B%5D context = new String%5Bpreds.size()%5D;%0A    preds.toArray(context);%0A    return context
+return preds
 ;%0A  
@@ -3833,13 +3833,15 @@
   pr
-ivate
+otected
  voi

Refactor the DefaultTokenContextGenerator to make it easier to create a sub-class