package br.ccsl.cogroo.analyzer;

import java.util.Arrays;
import java.util.List;

import opennlp.tools.postag.POSTaggerME;

import br.ccsl.cogroo.config.Analyzers;
import br.ccsl.cogroo.text.Document;
import br.ccsl.cogroo.text.Sentence;
import br.ccsl.cogroo.text.Token;
import br.ccsl.cogroo.text.impl.TokenImpl;
import br.ccsl.cogroo.util.TextUtils;

/**
 * The <code>POSTagger</code> class analyzes each token of a sentence and
 * classifies it grammatically.
 * 
 */
public class POSTagger implements AnalyzerI {
  private POSTaggerME tagger;

  public POSTagger(POSTaggerME tagger) {
    this.tagger = tagger;
  }

  public void analyze(Document document) {
    List<Sentence> sentences = document.getSentences();

    for (Sentence sentence : sentences) {
      List<Token> tokens = sentence.getTokens();
      String[] tags = tagger.tag(
          TextUtils.tokensToString(sentence.getTokens()), TextUtils
              .additionalContext(tokens, Arrays.asList(
                  Analyzers.CONTRACTION_FINDER, Analyzers.NAME_FINDER)));

      for (int i = 0; i < tags.length; i++) {
        ((TokenImpl) tokens.get(i)).setPOSTag(tags[i]);
      }
    }
  }
}

@@ -27,16 +27,44 @@
 lyzer;%0A%0A
+import java.util.ArrayList;%0A
 import j
@@ -80,16 +80,16 @@
 Arrays;%0A
-
 import j
@@ -145,17 +145,48 @@
 ggerME;%0A
+import opennlp.tools.util.Span;
 %0A
-
 import b
@@ -368,16 +368,56 @@
 enImpl;%0A
+import br.ccsl.cogroo.util.EntityUtils;%0A
 import b
@@ -1252,16 +1252,1736 @@
   %7D%0A    
-%7D
+  %0A      EntityUtils.groupTokens(sentence.getText(), tokens, createSpanList(toTokensArray(tokens), toTagsArray(tokens)));%0A    %7D%0A  %7D%0A  %0A  private String%5B%5D toTokensArray(List%3CToken%3E tokens) %7B%0A    String%5B%5D arr = new String%5Btokens.size()%5D;%0A    for (int i = 0; i %3C tokens.size(); i++) %7B%0A      arr%5Bi%5D = tokens.get(i).getLexeme();%0A    %7D%0A    return arr;%0A  %7D%0A%0A  private String%5B%5D toTagsArray(List%3CToken%3E tokens) %7B%0A    String%5B%5D arr = new String%5Btokens.size()%5D;%0A    for (int i = 0; i %3C tokens.size(); i++) %7B%0A      arr%5Bi%5D = tokens.get(i).getPOSTag();%0A    %7D%0A    return arr;%0A  %7D%0A%0A  // this is from opennlp%0A  public static List%3CSpan%3E createSpanList(String%5B%5D toks, String%5B%5D tags) %7B%0A%0A    // initialize with the list maximum size%0A    List%3CSpan%3E phrases = new ArrayList%3CSpan%3E(toks.length); %0A    String startTag = %22%22;%0A    int startIndex = 0;%0A    boolean foundPhrase = false;%0A%0A    for (int ci = 0, cn = tags.length; ci %3C cn; ci++) %7B%0A      String pred = tags%5Bci%5D;%0A      if(!tags%5Bci%5D.startsWith(%22B-%22) && !tags%5Bci%5D.startsWith(%22I-%22)) %7B%0A        pred = %22O%22;%0A      %7D%0A      if (pred.startsWith(%22B-%22)%0A          %7C%7C (!pred.equals(%22I-%22 + startTag) && !pred.equals(%22O%22))) %7B // start%0A        if (foundPhrase) %7B // handle the last%0A          phrases.add(new Span(startIndex, ci, startTag));%0A        %7D%0A        startIndex = ci;%0A        startTag = pred.substring(2);%0A        foundPhrase = true;%0A      %7D else if (pred.equals(%22I-%22 + startTag)) %7B // middle%0A        // do nothing%0A      %7D else if (foundPhrase) %7B// end%0A        phrases.add(new Span(startIndex, ci, startTag));%0A        foundPhrase = false;%0A        startTag = %22%22;%0A      %7D%0A    %7D%0A    if (foundPhrase) %7B // leftover%0A      phrases.add(new Span(startIndex, tags.length, startTag));%0A    %7D%0A%0A    return phrases;
 %0A  %7D%0A%7D%0A

Change POSTagger analyzer to group MWE. refs #598