package buzztaiki.lucene.lastuni;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.StringReader;

import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;

public class CJKLastUniGramFilterForSearchTest extends BaseTokenStreamTestCase {
    public void testSingleWord() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あいうえお")), false);
        assertTokenStreamContents(ts,
            new String[]{"あい", "いう", "うえ", "えお", ""}, 
            new int[]   {0,      1,      2,      3,      0},
            new int[]   {2,      3,      4,      5,      0},
            5
        );
    }
    public void testMultiWord() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あい う えお")), false);
        assertTokenStreamContents(ts,
            new String[]{"あい", "い", "う", "えお", ""}, 
            new int[]   {0,      1,    3,    5,      0},
            new int[]   {2,      2,    4,    7,      0},
            7
        );
    }

    public void testSingleChar() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あ")), false);
        assertTokenStreamContents(ts,
            new String[]{"あ"}, 
            new int[]   {0},
            new int[]   {1},
            1
        );
    }

    public void testSingleToken() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あい")), false);
        assertTokenStreamContents(ts,
            new String[]{"あい", ""}, 
            new int[]   {0,      0},
            new int[]   {2,      0},
            2
        );
    }

    public void testAsciiAndCJK() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あいabcうえお")), false);
        assertTokenStreamContents(ts,
            new String[]{"あい", "い", "abc", "うえ", "えお", ""}, 
            new int[]   {0,      1, 2, 5, 6, 0},
            new int[]   {2,      2, 5, 7, 8, 0},
            8
        );
    }

    public void testSingleTokenTwice() throws Exception {
        TokenStream ts = new CJKLastUniGramFilter(new CJKTokenizer(new StringReader("あい うえ")), false);
        assertTokenStreamContents(ts,
            new String[]{"あい", "い", "うえ", ""}, 
            new int[]   {0,      1,    3,      0},
            new int[]   {2,      2,    5,      0},
            5
        );
    }

}

@@ -1470,39 +1470,34 @@
 %22%E3%81%84%E3%81%86%22, %22%E3%81%86%E3%81%88%22, %22%E3%81%88%E3%81%8A%22
-, %22%22
 %7D,
- 
 %0A            new
@@ -1527,32 +1527,24 @@
    2,      3
-,      0
 %7D,%0A         
@@ -1580,32 +1580,24 @@
    4,      5
-,      0
 %7D,%0A         
@@ -1853,31 +1853,26 @@
 %22, %22%E3%81%86%22, %22%E3%81%88%E3%81%8A%22
-, %22%22
 %7D,
- 
 %0A           
@@ -1902,32 +1902,24 @@
 ,    3,    5
-,      0
 %7D,%0A         
@@ -1955,24 +1955,16 @@
  4,    7
-,      0
 %7D,%0A     
@@ -2209,17 +2209,16 @@
 %5B%5D%7B%22%E3%81%82%22%7D,
- 
 %0A       
@@ -2520,23 +2520,18 @@
 g%5B%5D%7B%22%E3%81%82%E3%81%84%22
-, %22%22
 %7D,
- 
 %0A       
@@ -2548,24 +2548,16 @@
 nt%5B%5D   %7B
-0,      
 0%7D,%0A    
@@ -2578,24 +2578,16 @@
 t%5B%5D   %7B2
-,      0
 %7D,%0A     
@@ -2863,23 +2863,18 @@
 %E3%81%88%22, %22%E3%81%88%E3%81%8A%22
-, %22%22
 %7D,
- 
 %0A       
@@ -2906,18 +2906,27 @@
  1, 
-2, 5, 6, 0
+   2,     5,      6
 %7D,%0A 
@@ -2964,18 +2964,27 @@
  2, 
-5, 7, 8, 0
+   5,     7,      8
 %7D,%0A 
@@ -3257,15 +3257,10 @@
 %22%E3%81%86%E3%81%88%22
-, %22%22
 %7D,
- 
 %0A   
@@ -3296,24 +3296,16 @@
  1,    3
-,      0
 %7D,%0A     
@@ -3343,16 +3343,8 @@
    5
-,      0
 %7D,%0A 

Remove last empty string in test.