package org.lionsoul.jcseg.analyzer;

import java.io.IOException;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.lionsoul.jcseg.tokenizer.core.ADictionary;
import org.lionsoul.jcseg.tokenizer.core.ISegment;
import org.lionsoul.jcseg.tokenizer.core.IWord;
import org.lionsoul.jcseg.tokenizer.core.JcsegException;
import org.lionsoul.jcseg.tokenizer.core.JcsegTaskConfig;
import org.lionsoul.jcseg.tokenizer.core.SegmentFactory;


/**
 * <p>
 * here is the documentation from {@link org.apache.lucene.analysis.tokenizer}
 * A Tokenizer is a TokenStream whose input is a Reader.
 * </p>
 * 
 * <p>
 * This is an abstract class; subclasses must override {@link #incrementToken()}
 * <p>
 * 
 * <p>
 * NOTE: Subclasses overriding {@link #incrementToken()} must
 * call {@link AttributeSource#clearAttributes()} before setting attributes
 * </p>
 *
 * <p>
 * lucene invoke Tokenizer#setReader(Reader input) to set the inputPending
 * after invoke the reset, global object input will be available
 * </p>
 * 
 * <p>jcseg tokennizer for lucene on or after 5.1.0</p>
 * 
 * @author    chenxin<chenxin619315@gmail.com>
 */
public class JcsegTokenizer extends Tokenizer 
{
    //the default jcseg segmentor
    private ISegment segmentor;

    private final CharTermAttributeImpl termAtt = (CharTermAttributeImpl)addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
    
    public JcsegTokenizer(
        int mode,
        JcsegTaskConfig config,
        ADictionary dic ) throws JcsegException, IOException 
    {
        segmentor = SegmentFactory.createJcseg(mode, new Object[]{config, dic});
        segmentor.reset(input);
    }

    @Override
    final public boolean incrementToken() throws IOException 
    {
        IWord word = segmentor.next();
        if ( word == null ) {
            return false;
        }
        
        //char[] token = word.getValue().toCharArray();
        //termAtt.copyBuffer(token, 0, token.length);
        
        termAtt.clear();
        termAtt.append(word.getValue());
        offsetAtt.setOffset(word.getPosition(), word.getPosition() + word.getLength());
        typeAtt.setType("word");
        
        return true;
    }
    
    @Override
    public void reset() throws IOException 
    {
        super.reset();
        segmentor.reset(input);
        clearAttributes();
    }
}

@@ -1829,24 +1829,187 @@
 ss);%0D%0A    %0D%0A
+    /**%0D%0A     * field level offset tracker for multiple-value field%0D%0A     * like the Array field in Elasticseach %0D%0A    */%0D%0A    private int fieldOffset = 0;%0D%0A    %0D%0A
     public J
@@ -2582,24 +2582,85 @@
 n.length);%0D%0A
+        int startOffset = fieldOffset + word.getPosition();%0D%0A
         %0D%0A  
@@ -2757,46 +2757,32 @@
 set(
-word.getPosition(), word.getPosition()
+startOffset, startOffset
  + w
@@ -2875,24 +2875,204 @@
    %7D%0D%0A    %0D%0A
+    /**%0D%0A     * reset the field-level offset %0D%0A    */%0D%0A    @Override%0D%0A    public void end() throws IOException%0D%0A    %7B%0D%0A        super.end();%0D%0A        fieldOffset = 0;%0D%0A    %7D%0D%0A    %0D%0A
     @Overrid
@@ -3218,11 +3218,17 @@
 %0A    %7D%0D%0A
+    %0D%0A
 %7D%0D%0A

Add field level multiple-value offset support