package uk.ac.susx.tag.classificationframework.featureextraction.filtering;

/*
 * #%L
 * TokenFilterPunctuation.java - classificationframework - CASM Consulting - 2,013
 * %%
 * Copyright (C) 2013 - 2014 CASM Consulting
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.google.common.collect.Sets;
import uk.ac.susx.tag.classificationframework.datastructures.Document;

import java.util.Set;
import java.util.regex.Pattern;

/**
 * This is a TokenFilter, see the TokenFilter abstract class for details of function and purpose of
 * token filters.
 *
 * This filter filters out punctuation according to a regex of what constitutes punctuation.
 * It's an alternative to using the PoS to filter punctuation.
 *
 * User: Andrew D. Robertson
 * Date: 19/08/2013
 * Time: 12:11
 */
public class TokenFilterPunctuation extends TokenFilter {

    private static final long serialVersionUID = 0L;

    private Pattern puncPattern;
    private boolean filterExclamationAndQuestionMarks;

    public TokenFilterPunctuation(boolean filterExclamationAndQuestionMarks){
        this.filterExclamationAndQuestionMarks = filterExclamationAndQuestionMarks;
        puncPattern = getPunctuationPattern();
    }

    private Pattern getPunctuationPattern() {
        return filterExclamationAndQuestionMarks?
                Pattern.compile("[!?\"#$%&'()*+,-./:;<=>@\\[\\]^_`{|}~]+"):
                Pattern.compile("[\"#$%&'()*+,-./:;<=>@\\[\\]^_`{|}~]+");
    }

    public TokenFilterPunctuation() {
        this(false);
    }

    public boolean filter(int index, Document tokens) {
        return puncPattern.matcher(tokens.get(index).get("form")).matches();
    }

    @Override
    public boolean isThreadSafe() {
        return true;
    }
}

@@ -1822,16 +1822,18 @@
 nMarks?%0A
+//
         
@@ -1900,16 +1900,18 @@
 %7D~%5D+%22):%0A
+//
         
@@ -1976,16 +1976,194 @@
 %7D~%5D+%22);%0A
+                Pattern.compile(%22%5B!%EF%BC%81?%EF%BC%9F%5C%22%E2%80%9C%E2%80%9D#$%25&'()%EF%BC%88%EF%BC%89%E3%80%8C%E3%80%8D%E3%80%90%E3%80%91%E3%80%81*+%EF%BC%8C,-.%E3%80%82/:%EF%BC%9A%EF%BC%9B;%3C=%3E@%5C%5C%5B%5C%5C%5D%5E_%60%7B%7C%7D~%5D+%22):%0A                Pattern.compile(%22%5B%5C%22%E2%80%9C%E2%80%9D#$%25&'()%EF%BC%88%EF%BC%89%E3%80%8C%E3%80%8D%E3%80%90%E3%80%91%E3%80%81*+%EF%BC%8C,-.%E3%80%82/:%EF%BC%9A%EF%BC%9B;%3C=%3E@%5C%5C%5B%5C%5C%5D%5E_%60%7B%7C%7D~%5D+%22);%0A
     %7D%0A%0A 

update to allow Chinese punctuation