package buzztaiki.lucene.lastuni;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.StringReader;

import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;

public class CJKLastUniGramFilterTest extends BaseTokenStreamTestCase {
    public void testSingleWord() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あいうえお")));
        assertTokenStreamContents(ts,
            new String[]{"あい", "いう", "うえ", "えお", "お"}, 
            new int[]   {0,      1,      2,      3,      4},
            new int[]   {2,      3,      4,      5,      5},
            5
        );
    }
    public void testMultiWord() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あい う えお")));
        assertTokenStreamContents(ts,
            new String[]{"あい", "い", "う", "えお", "お"}, 
            new int[]   {0,      1,    3,    5,      6},
            new int[]   {2,      2,    4,    7,      7},
            7
        );
    }

    public void testSingleChar() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あ")));
        assertTokenStreamContents(ts,
            new String[]{"あ"}, 
            new int[]   {0},
            new int[]   {1},
            1
        );
    }

    public void testSingleToken() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あい")));
        assertTokenStreamContents(ts,
            new String[]{"あい", "い"}, 
            new int[]   {0,      1},
            new int[]   {2,      2},
            2
        );
    }
}

@@ -2641,11 +2641,393 @@
 );%0A    %7D
+%0A%0A    public void testAsciiAndCJK() throws Exception %7B%0A        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader(%22%E3%81%82%E3%81%84abc%E3%81%86%E3%81%88%E3%81%8A%22)));%0A        assertTokenStreamContents(ts,%0A            new String%5B%5D%7B%22%E3%81%82%E3%81%84%22, %22%E3%81%84%22, %22abc%22, %22%E3%81%86%E3%81%88%22, %22%E3%81%88%E3%81%8A%22, %22%E3%81%8A%22%7D, %0A            new int%5B%5D   %7B0,      1, 2, 5, 6, 7%7D,%0A            new int%5B%5D   %7B2,      2, 5, 7, 8, 8%7D,%0A            8%0A        );%0A    %7D
 %0A%7D%0A

Add test