/*
 This software was produced for the U. S. Government
 under Contract No. W15P7T-11-C-F600, and is
 subject to the Rights in Noncommercial Computer Software
 and Noncommercial Computer Software Documentation
 Clause 252.227-7014 (JUN 1995)

 Copyright 2013 The MITRE Corporation. All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

package org.opensextant.solrtexttagger;

import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.shingle.ShingleFilter;
import org.apache.lucene.analysis.tokenattributes.*;

import java.io.IOException;

/**
 * Concatenate all tokens, separated by a provided character,
 * defaulting to a single space. It always produces exactly one token, and it's designed to be the
 * last token filter in an analysis chain.
 *
 * @author David Smiley - dsmiley@mitre.org
 */
public class ConcatenateFilter extends TokenFilter {

  /*
  For a very different approach that could accept synonyms or anything except position gaps (e.g.
  not stopwords),
  consider using o.a.l.analysis.TokenStreamToAutomaton
  with o.a.l.util.automaton.SpecialOperations.getFiniteStrings().
  For gaps (stopwords), we could perhaps index a special token at those gaps and then have the
  tagger deal with them -- also doable.
   */

  private char separator = ' ';

  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
  private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
  private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);

  private boolean done;
  private StringBuilder buf = new StringBuilder(128);

  /**
   * Construct a token stream filtering the given input.
   */
  protected ConcatenateFilter(TokenStream input) {
    super(input);
  }

  public void setTokenSeparator(char separator) {
    this.separator = separator;
  }

  @Override
  public void reset() throws IOException {
    input.reset();
    done = false;
  }

  @Override
  public final boolean incrementToken() throws IOException {
    if (done)
      return false;
    done = true;

    buf.setLength(0);
    boolean firstTerm = true;
    while (input.incrementToken()) {
      int posInc = posIncrAtt.getPositionIncrement();
      if (firstTerm)
        posInc--;
      for (int i = 0; i < posInc; i++) {
        buf.append(separator);
      }
      buf.append(termAtt);
      firstTerm = false;
    }
    input.end();

    termAtt.setEmpty().append(buf);
    //Setting the other attributes ultimately won't have much effect but lets be thorough
    offsetAtt.setOffset(0, offsetAtt.endOffset());
    posIncrAtt.setPositionIncrement(1);
    posLenAtt.setPositionLength(1);//or do we add up the positions?  Probably not used any way.
    typeAtt.setType(ShingleFilter.DEFAULT_TOKEN_TYPE);//"shingle"

    return true;
  }

  @Override
  public void end() throws IOException {
    //we already called input.end() in incrementToken
  }
}

@@ -2994,16 +2994,98 @@
 sInc--;%0A
+      //Note: it's debatable what to do when posInc %3E 1 (stopword); see issue #13%0A
       fo

Comment on posInc > 1 (stopwords); issue #13