package de.moritzf.nlp.jtopia.cleaner;

import java.util.ArrayList;
import java.util.List;

import java.util.regex.Matcher;
import java.util.regex.Pattern;


import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

/**
 * The type Text cleaner.
 *
 * @author : sree
 *
 * <p>
 * Normalizes and tokenizes the input text
 */
public class TextCleaner {

    private static final Logger logger = LogManager.getLogger(TextCleaner.class);

    private static final String UMLAUT = "äåüöÄÅÖÜßæÆøØáéíóúÁÉÍÑÓÚéÉèÈàÀùÙâêîôûÂÊÎÔÛçÇãÃáÁàÀâÂéÉêÊíÍõÕóÓôÔúÚüÜ";

    private static final String MATCH_PATTERN = "([^a-zA-Z" + UMLAUT + "]*)([a-z" + UMLAUT + "A-Z-\\.]*[a-zA-Z"
            + UMLAUT + "])([^a-zA-Z" + UMLAUT + "]*[a-zA-Z" + UMLAUT + "]*)";

    /**
     * Normalizing the new line spaces in input text
     *
     * @param text the text
     * @return string
     */
    public String normalizeText(String text) {
        logger.debug("Input to normalize text : " + text);
        if (text != null && !text.isEmpty()) {
            text = replaceAll(text, "\\n", " . ");
        }
        logger.debug("Input text normalized : " + text);
        return text.trim();
    }

    /**
     * Tokenizing the text using regex
     *
     * @param text the text
     * @return list
     */
    public List<String> tokenizeText(String text) {
        List<String> tokenizedWords = new ArrayList<String>();
        String[] words = text.split("\\s");
        Pattern pattern = Pattern.compile(MATCH_PATTERN, Pattern.DOTALL | Pattern.MULTILINE);

        for (String word : words) {
            // If the term is empty, skip it, since we probably just have multiple whitespace characters.
            if (word == null || word.isEmpty())
                continue;
            // Now, a word can be preceded or succeeded by symbols, so let's split those out
            Matcher matcher = pattern.matcher(word);
            if (!matcher.find()) {
                tokenizedWords.add(word);
                continue;
            }
            for (int i = 1; i <= matcher.groupCount(); i++) {
                if (matcher.group(i) != null && !matcher.group(i).isEmpty()) {
                    logger.debug("Matcher group : " + i + " text :  " + matcher.group(i));
                    tokenizedWords.add(matcher.group(i));
                }
            }
        }

        return tokenizedWords;

    }

    /**
     * Replace all the occurances of a regex with replacement string
     *
     * @param text
     * @param regex
     * @param replacement
     * @return the text after replacements
     */

    private String replaceAll(String text, String regex, String replacement) {
        text = text.replaceAll(regex, replacement);
        return text;
    }

}

@@ -521,18 +521,16 @@
 %C3%B3%C3%BA%C3%81%C3%89%C3%8D%C3%91%C3%93%C3%9A
-%C3%A9%C3%89
 %C3%A8%C3%88%C3%A0%C3%80%C3%B9%C3%99%C3%A2%C3%AA
@@ -545,30 +545,10 @@
 %C3%A7%C3%87%C3%A3%C3%83
-%C3%A1%C3%81%C3%A0%C3%80%C3%A2%C3%82%C3%A9%C3%89%C3%AA%C3%8A%C3%AD%C3%8D%C3%B5%C3%95%C3%B3%C3%93%C3%B4%C3%94%C3%BA%C3%9A%C3%BC%C3%9C
+%C3%B5%C3%95
 %22;%0A%0A

Remove redundant chars