package com.worksap.labs.solr.helper;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.lucene.analysis.Token;

import java.util.List;
import java.util.Map;

public final class TokenStreamBuildHelper {

	public static Map<Integer, List<Token>> build(Map<Integer, List<Token>> source) {
		Map<Integer, List<Token>> target = deepClone(source);
		Map<Token, Integer> filterMap = generateFilterMap(source);
		for (Map.Entry<Integer, List<Token>> entry : source.entrySet()) {
			int pos = entry.getKey();
			List<Token> tokens = entry.getValue();
			for (Token token : tokens) {
				for (Map.Entry<Token, Integer> filter : filterMap.entrySet()) {
					int filterPos = filter.getValue();
					Token filterToken = filter.getKey();
					if (pos <= filterPos) {
						break;
					}
					if (filterToken.toString().indexOf(token.toString()) != -1) {
						target.get(filterPos).add(token);
						target.get(pos).remove(token);
						if (target.get(pos).isEmpty()) {
							target.remove(pos);
						}
					}
				}
			}
		}
		return target;
	}

	private static Map<Integer, List<Token>> deepClone(Map<Integer, List<Token>> source) {
		Map<Integer, List<Token>> target = Maps.newTreeMap();
		for (Map.Entry<Integer, List<Token>> entry : source.entrySet()) {
			target.put(entry.getKey(), Lists.newArrayList(entry.getValue()));
		}
		return target;
	}

	private static boolean isEnglishWord(Token token) {
		boolean isEnglishWord = false;
		for (int i = 0; i < token.length(); i++) {
			if (0x40 < token.buffer()[i] && token.buffer()[i] < 0x7b) {
				isEnglishWord = true;
				break;
			}
		}
		return isEnglishWord;
	}

	private static Map<Token, Integer> generateFilterMap(Map<Integer, List<Token>> source) {
		Map<Token, Integer> filterMap = Maps.newHashMap();

		for (Map.Entry<Integer, List<Token>> entry : source.entrySet()) {
			int pos = entry.getKey();
			List<Token> tokens = entry.getValue();

			for (Token token : tokens) {
				if (token.length() > 1 && !isEnglishWord(token)) {
					filterMap.put(token, pos);
				}
			}
		}

		return filterMap;
	}
}

@@ -153,16 +153,45 @@
 Token;%0A%0A
+import java.util.LinkedList;%0A
 import j
@@ -201,24 +201,24 @@
 .util.List;%0A
-
 import java.
@@ -348,32 +348,68 @@
 ken%3E%3E source) %7B%0A
+%09%09source = removeDuplicate(source);%0A
 %09%09Map%3CInteger, L
@@ -1132,32 +1132,632 @@
 urn target;%0A%09%7D%0A%0A
+%09private static Map%3CInteger, List%3CToken%3E%3E removeDuplicate(Map%3CInteger, List%3CToken%3E%3E source) %7B%0A%09%09Map%3CInteger, List%3CToken%3E%3E target = Maps.newTreeMap();%0A%0A%09%09Map%3CString, Integer%3E counter = Maps.newHashMap();%0A%0A%09%09for (int pos : source.keySet()) %7B%0A%09%09%09List%3CToken%3E tokens = source.get(pos);%0A%09%09%09if (!target.containsKey(pos)) %7B%0A%09%09%09%09target.put(pos, new LinkedList%3CToken%3E());%0A%09%09%09%7D%0A%09%09%09for (Token token : tokens) %7B%0A%09%09%09%09Integer num = counter.get(token.toString());%0A%09%09%09%09if (null == num) %7B%0A%09%09%09%09%09target.get(pos).add(token);%0A%09%09%09%09%09counter.put(token.toString(), 1);%0A%09%09%09%09%7D%0A%09%09%09%7D%0A%09%09%09counter.clear();%0A%09%09%7D%0A%0A%09%09return target;%0A%09%7D%0A%0A
 %09private static 
@@ -2448,32 +2448,87 @@
 ps.newHashMap();
+%0A%09%09Map%3CString, Token%3E tokenMapping = Maps.newHashMap();
 %0A%0A%09%09for (Map.Ent
@@ -2736,24 +2736,151 @@
 d(token)) %7B%0A
+%09%09%09%09%09Token tmp = tokenMapping.get(token.toString());%0A%09%09%09%09%09if (null == tmp) %7B%0A%09%09%09%09%09%09tokenMapping.put(token.toString(), token);%0A%09
 %09%09%09%09%09filterM
@@ -2895,24 +2895,31 @@
 oken, pos);%0A
+%09%09%09%09%09%7D%0A
 %09%09%09%09%7D%0A%09%09%09%7D%0A%09

remove duplicate tokens