package org.lionsoul.jcseg.analyzer;

import java.io.IOException;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.lionsoul.jcseg.ISegment;
import org.lionsoul.jcseg.IWord;
import org.lionsoul.jcseg.dic.ADictionary;
import org.lionsoul.jcseg.segmenter.SegmenterConfig;


/**
 * <p>
 * here is the documentation from {@link org.apache.lucene.analysis.Tokenizer}
 * A Tokenizer is a TokenStream whose input is a Reader.
 * </p>
 * 
 * <p>
 * This is an abstract class; subclasses must override {@link #incrementToken()}
 * <p>
 * 
 * <p>
 * NOTE: Subclasses overriding {@link #incrementToken()} must
 * call {@link #clearAttributes()} before setting attributes
 * </p>
 *
 * <p>
 * lucene invoke Tokenizer#setReader(Reader input) to set the inputPending
 * after invoke the reset, global object input will be available
 * </p>
 * 
 * <p>jcseg tokennizer for lucene on or after 5.1.0</p>
 * 
 * @author    chenxin<chenxin619315@gmail.com>
 */
public class JcsegTokenizer extends Tokenizer 
{
    // The default Jcseg segmentor
    private final ISegment segmentor;

    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
    
    private int endPosition;
    
    public JcsegTokenizer(
        ISegment.Type type,
        SegmenterConfig config,
        ADictionary dic )
    {
        segmentor = type.factory.create(config, dic);
    }

    @Override
    final public boolean incrementToken() throws IOException 
    {
    	/* Clear the attributes */
    	clearAttributes();
    	
        final IWord word = segmentor.next();
        if ( word == null ) {
            return false;
        }
        
        // final char[] token = word.getValue().toCharArray();
        // termAtt.copyBuffer(token, 0, token.length);
        termAtt.append(word.getValue());
        termAtt.setLength(word.getLength());
        offsetAtt.setOffset(
        	correctOffset(word.getPosition()), 
        	correctOffset(word.getPosition() + word.getLength())
        );
        endPosition = offsetAtt.endOffset();
        typeAtt.setType("word");
        
        return true;
    }
    
    @Override
    public void end() throws IOException
    {
        super.end();
        final int finalOffset = correctOffset(endPosition);
        offsetAtt.setOffset(finalOffset, finalOffset);
    }
    
    @Override
    public void reset() throws IOException 
    {
        super.reset();
        segmentor.reset(input);
    }
    
}

@@ -1570,31 +1570,165 @@
     
-private int endPosition
+/**%0D%0A     * field level offset tracker for multiple-value field%0D%0A     * like the Array field in Elasticseach or Solr%0D%0A    */%0D%0A    private int fieldOffset = 0
 ;%0D%0A 
@@ -2143,24 +2143,74 @@
 = null ) %7B%0D%0A
+            fieldOffset = offsetAtt.endOffset();%0D%0A
             
@@ -2255,22 +2255,16 @@
       //
- final
  char%5B%5D 
@@ -2492,32 +2492,46 @@
  %09correctOffset(
+fieldOffset + 
 word.getPosition
@@ -2560,16 +2560,30 @@
 tOffset(
+fieldOffset + 
 word.get
@@ -2630,54 +2630,8 @@
 );%0D%0A
-        endPosition = offsetAtt.endOffset();%0D%0A
     
@@ -2841,19 +2841,19 @@
 set(
-endPosition
+fieldOffset
 );%0D%0A
@@ -2904,24 +2904,55 @@
 alOffset);%0D%0A
+        this.fieldOffset = 0;%0D%0A
     %7D%0D%0A    %0D

Optimize the tokenstream attributes track