/*
 * Copyright 2011-2015 B2i Healthcare Pte Ltd, http://b2i.sg
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.b2international.snowowl.datastore.index;

import java.io.Reader;

import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.Version;

import com.b2international.collections.set.IntSet;
import com.b2international.commons.collect.PrimitiveSets;
import com.b2international.snowowl.core.TextConstants;

/**
 * A character-oriented tokenizer which splits tokens on whitespace and delimiters enumerated in
 * {@link IndexUtils#DELIMITERS}, and also converts characters to lower case in the normalization phase.
 * 
 */
public class DelimiterTokenizer extends CharTokenizer {

	// Excludes 2000-2000a, which is handled as a range
	private static final String BREAKING_WHITESPACE_CHARS = "\t\n\013\f\r \u0085\u1680\u2028\u2029\u205f\u3000";

	// Excludes 2007, which is handled as a gap in a pair of ranges
	private static final String NON_BREAKING_WHITESPACE_CHARS = "\u00a0\u180e\u202f";
	
	/**
	 * @see TextConstants#WHITESPACE_OR_DELIMITER_MATCHER
	 */
	private static final IntSet TOKEN_CHARS;
	
	static {
		
		final IntSet set = PrimitiveSets.newIntOpenHashSet();
		
		char[] charArray = TextConstants.DELIMITERS.toCharArray();
		for (int i = 0; i < charArray.length; i++) {
			set.add(charArray[i]);
		}
		
		charArray = BREAKING_WHITESPACE_CHARS.toCharArray();
		for (int i = 0; i < charArray.length; i++) {
			set.add(charArray[i]);
		}
		
		charArray = NON_BREAKING_WHITESPACE_CHARS.toCharArray();
		for (int i = 0; i < charArray.length; i++) {
			set.add(charArray[i]);
		}
		
		final int first = '\u2000';
		final int last = '\u200a';
		
		for (int i = first; i < last; i++) {
			set.add(i);
		}
		
		set.trimToSize();
		
		TOKEN_CHARS = PrimitiveSets.newUnmodifiableIntSet(set);
		
	}
	
	public DelimiterTokenizer(Reader input) {
		super(Version.LUCENE_4_9, input);
	}
	
	@Override
	protected int normalize(int c) {
		return Character.toLowerCase(c);
	}
	
	@Override
	protected boolean isTokenChar(int c) {
		// We don't have whitespace characters to match in the supplementary code point range
		return c >= Character.MIN_SUPPLEMENTARY_CODE_POINT || !TOKEN_CHARS.contains(c);
	}
}

@@ -824,143 +824,77 @@
 nal.
-collections.set.IntSet;%0Aimport com.b2international.commons.collect.PrimitiveSets;%0Aimport com.b2international.snowowl.core.TextConstants
+snowowl.core.TextConstants;%0Aimport com.google.common.base.CharMatcher
 ;%0A%0A/
@@ -1564,22 +1564,27 @@
 c final 
-IntSet
+CharMatcher
  TOKEN_C
@@ -1607,689 +1607,270 @@
 %7B%0A%09%09
-%0A%09%09final IntSet set = PrimitiveSets.newIntOpenHashSet();%0A%09%09%0A%09%09char%5B%5D charArray = TextConstants.DELIMITERS.toCharArray();%0A%09%09for (int i = 0; i %3C charArray.length; i++) %7B
+TOKEN_CHARS = CharMatcher.ANY
 %0A%09%09%09
-set
 .a
-dd(charArray%5Bi%5D);%0A%09%09%7D%0A%09%09%0A%09%09charArray = BREAKING_WHITESPACE_CHARS.toCharArray();%0A%09%09for (int i = 0; i %3C charArray.length; i++) %7B
+nd(CharMatcher.anyOf(BREAKING_WHITESPACE_CHARS))
 %0A%09%09%09
-set
 .a
-dd(charArray%5Bi%5D);%0A%09%09%7D%0A%09%09%0A%09%09charArray = NON_BREAKING_WHITESPACE_CHARS.toCharArray();%0A%09%09for (int i = 0; i %3C charArray.length; i++) %7B
+nd(CharMatcher.anyOf(NON_BREAKING_WHITESPACE_CHARS))%0A%09%09%09.and(CharMatcher.anyOf(TextConstants.DELIMITERS))
 %0A%09%09%09
-set
 .a
-dd(charArray%5Bi%5D);%0A%09%09%7D%0A%09%09%0A%09%09final int first = '%5Cu2000';%0A%09%09final int last = '%5Cu200a';%0A%09%09%0A%09%09for (int i = first; i %3C last; i++) %7B%0A%09%09%09set.add(i);%0A%09%09%7D%0A%09%09%0A%09%09set.trimToSize();%0A%09%09%0A%09%09TOKEN_CHARS = PrimitiveSets.newUnmodifiableIntSet(set);%0A%09%09
+nd(CharMatcher.inRange('%5Cu2000', '%5Cu200a'))%0A%09%09%09.precomputed();
 %0A%09%7D%0A
@@ -2252,17 +2252,23 @@
 ARS.
-contains(
+matches((char) 
 c);%0A

use preComputed CharMatcher instead of IntSet...