package crawler.util.analyzer;

import java.io.IOException;

import org.apache.lucene.analysis.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;


public class TwitterFilter extends FilteringTokenFilter {
	//private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
	private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
	
	public TwitterFilter(boolean enablePositionIncrements, TokenStream input) {
		super(enablePositionIncrements, input);
	}

	@Override
	protected boolean accept() throws IOException {
		String type = typeAtt.type();
		if (type.equals(TwitterAttri.PONCTUATIONS)||type.equals(TwitterAttri.MENTION)||type.equals(TwitterAttri.URL))
			return false;
		else return true;
	}

}

@@ -1,8 +1,593 @@
+/* Copyright %5B2014%5D %5BXinyue Wang%5D%0A   Licensed under the Apache License, Version 2.0 (the %22License%22);%0A   you may not use this file except in compliance with the License.%0A   You may obtain a copy of the License at%0A     http://www.apache.org/licenses/LICENSE-2.0%0A   Unless required by applicable law or agreed to in writing, software%0A   distributed under the License is distributed on an %22AS IS%22 BASIS,%0A   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A   See the License for the specific language governing permissions and%0A   limitations under the License.%0A*/%0A%0A
 package 

Update TwitterFilter.java