package crawler.util.analyzer;

import java.io.Reader;

//import org.apache.lucene.analysis.CharTokenizer;
import org.apache.lucene.analysis.LetterTokenizer;
import org.apache.lucene.util.Version;

public class TwitterLetterDigitTokenizer extends LetterTokenizer{
	
	public TwitterLetterDigitTokenizer(Version matchVersion, Reader in) {
		super(matchVersion, in);
	}

	protected boolean  isTokenChar(int c) {
		if ((c==(int)'@')||(c==(int)'#')||(c==(int)'/')||(c==(int)':')||(c==(int)'.'))
			return true;
		else return Character.isLetterOrDigit(c);
	}

	
}

@@ -1,8 +1,593 @@
+/* Copyright %5B2014%5D %5BXinyue Wang%5D%0A   Licensed under the Apache License, Version 2.0 (the %22License%22);%0A   you may not use this file except in compliance with the License.%0A   You may obtain a copy of the License at%0A     http://www.apache.org/licenses/LICENSE-2.0%0A   Unless required by applicable law or agreed to in writing, software%0A   distributed under the License is distributed on an %22AS IS%22 BASIS,%0A   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A   See the License for the specific language governing permissions and%0A   limitations under the License.%0A*/%0A%0A
 package 

Update TwitterLetterDigitTokenizer.java