package org.lemurproject.galago.core.parse.tagtok;

/**
 * @author trevor.
 * @see org.lemurproject.galago.core.parse.TagTokenizer
 */
public class TagPunctuation {

	static final char[] splitChars = {
	  ' ', '\t', '\n', '\r', // spaces
		';', '\"', '&', '/', ':', '!', '#',
		'?', '$', '%', '(', ')', '@', '^',
		'*', '+', '-', ',', '=', '>', '<', '[',
		']', '{', '}', '|', '`', '~', '_'
	};

	public static final boolean[] splits;
	static {
		splits = TagPunctuation.buildSplits();
	}

	public static boolean[] buildSplits() {
		boolean[] localSplits = new boolean[257];

		for (int i = 0; i < localSplits.length; i++) {
			localSplits[i] = false;
		}

		for (char c : splitChars) {
			localSplits[(byte) c] = true;
		}

		for (byte c = 0; c <= 32; c++) {
			localSplits[c] = true;
		}

		return localSplits;
	}
}

@@ -808,11 +808,279 @@
 lits;%0A%09%7D
+%0A%0A%09public static String clean(String input) %7B%0A%09%09StringBuilder output = new StringBuilder();%0A%09%09for (char c : input.toCharArray()) %7B%0A%09%09%09if(c %3C splits.length && splits%5Bc%5D) %7B%0A%09%09%09%09continue;%0A%09%09%09%7D%0A%09%09%09output.append(Character.toLowerCase(c));%0A%09%09%7D%0A%09%09return output.toString();%0A%09%7D
 %0A%7D%0A

clean function from TagTokenizer