package de.danielnaber.languagetool.dev.index;

import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;

import de.danielnaber.languagetool.AnalyzedSentence;
import de.danielnaber.languagetool.AnalyzedToken;
import de.danielnaber.languagetool.AnalyzedTokenReadings;
import de.danielnaber.languagetool.JLanguageTool;

public class LanguageToolFilter extends TokenFilter {
  private JLanguageTool lt;

  private Iterator<AnalyzedTokenReadings> tokenIter;

  private List<AnalyzedTokenReadings> tokenBuffer;

  private CharTermAttribute termAtt;

  private OffsetAttribute offsetAtt;

  private PositionIncrementAttribute posIncrAtt;

  private TypeAttribute typeAtt;

  private AttributeSource.State current;

  public static final String POS_PREFIX = "_POS_";

  protected LanguageToolFilter(TokenStream input, JLanguageTool lt) {
    super(input);
    this.lt = lt;
    termAtt = addAttribute(CharTermAttribute.class);
    offsetAtt = addAttribute(OffsetAttribute.class);
    posIncrAtt = addAttribute(PositionIncrementAttribute.class);
    typeAtt = addAttribute(TypeAttribute.class);
  }

  @Override
  public boolean incrementToken() throws IOException {

    if (tokenIter == null || !tokenIter.hasNext()) {
      // there are no remaining tokens from the current sentence... are there more sentences?
      if (input.incrementToken()) {
        // a new sentence is available: process it.
        AnalyzedSentence sentence = lt.getAnalyzedSentence(termAtt.toString());

        tokenBuffer = Arrays.asList(sentence.getTokens());
        tokenIter = tokenBuffer.iterator();
        /*
         * it should not be possible to have a sentence with 0 words, check just in case. returning
         * EOS isn't the best either, but its the behavior of the original code.
         */
        if (!tokenIter.hasNext())
          return false;
      } else {
        return false; // no more sentences, end of stream!
      }
    }

    // It must clear attributes, as it is creating new tokens.
    clearAttributes();
    AnalyzedTokenReadings tr = tokenIter.next();
    AnalyzedToken at = tr.getAnalyzedToken(0);

    if (tr.isWhitespace()) {
      return this.incrementToken();
    }

    offsetAtt.setOffset(tr.getStartPos(), tr.getStartPos() + at.getToken().length());

    for (int i = 0; i < tr.getReadingsLength(); i++) {
      at = tr.getAnalyzedToken(i);
    }

    current = captureState();
    termAtt.append(tr.getAnalyzedToken(0).getToken());

    return true;
  }

}

@@ -149,16 +149,41 @@
 l.List;%0D
+%0Aimport java.util.Stack;%0D
 %0A%0D%0Aimpor
@@ -971,55 +971,55 @@
 %0D%0A  
-private List%3CAnalyzedTokenRead
+// stack for POS%0D%0A  private Stack%3CStr
 ing
-s
 %3E 
-tokenBuffer
+stack
 ;%0D%0A%0D
@@ -1394,16 +1394,50 @@
  = lt;%0D%0A
+    stack = new Stack%3CString%3E();%0D%0A
     term
@@ -1726,24 +1726,247 @@
 eption %7B%0D%0A%0D%0A
+    if (stack.size() %3E 0) %7B%0D%0A      String pop = stack.pop();%0D%0A      restoreState(current);%0D%0A      termAtt.append(pop);%0D%0A      posIncrAtt.setPositionIncrement(0);%0D%0A      typeAtt.setType(%22pos%22);%0D%0A      return true;%0D%0A    %7D%0D%0A%0D%0A
     if (toke
@@ -2278,24 +2278,52 @@
 ;%0D%0A%0D%0A       
+ List%3CAnalyzedTokenReadings%3E
  tokenBuffer
@@ -2954,24 +2954,228 @@
 oken(0);%0D%0A%0D%0A
+    // add POS tag for sentence start.%0D%0A    if (tr.isSentStart()) %7B%0D%0A      termAtt.append(POS_PREFIX + tr.getAnalyzedToken(0).getPOSTag());%0D%0A      return true;%0D%0A    %7D%0D%0A%0D%0A    // by pass the white spaces.%0D%0A
     if (tr.i
@@ -3415,24 +3415,120 @@
 dToken(i);%0D%0A
+      if (at.getPOSTag() != null) %7B%0D%0A        stack.push(POS_PREFIX + at.getPOSTag());%0D%0A      %7D%0D%0A
     %7D%0D%0A%0D%0A   
@@ -3631,18 +3631,18 @@
  true;%0D%0A
+%0D%0A
   %7D%0D%0A
-%0D%0A
 %7D%0D%0A

index AnalyzedSentence with POS