package org.atlasapi.util;

import java.io.IOException;
import java.io.StringReader;
import java.util.List;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;

import com.google.common.base.Joiner;
import com.google.common.collect.Lists;

public class Strings {

    public static List<String> tokenize(String value, boolean filterStopWords) {
        List<String> tokensAsStrings = Lists.newArrayList();
        try {
            TokenStream tokens;
            if (filterStopWords) {
                tokens = new StandardAnalyzer(Version.LUCENE_42)
                    .tokenStream("", new StringReader(value));
            } else {
                tokens = new StandardAnalyzer(Version.LUCENE_42, CharArraySet.EMPTY_SET)
                    .tokenStream("", new StringReader(value));
            }
            tokens.reset();
            while (tokens.incrementToken()) {
                CharTermAttribute token = tokens.getAttribute(CharTermAttribute.class);
                tokensAsStrings.add(token.toString());
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        if (tokensAsStrings.isEmpty() && filterStopWords) {
            return tokenize(value, false);
        } else {
            return tokensAsStrings;
        }
    }

    public static String flatten(String value) {
        return Joiner.on("").join(tokenize(value, true)).replaceAll("[^a-zA-Z0-9]", "").toLowerCase();
    }
}

@@ -753,24 +753,21 @@
 ersion.L
-UCENE_42
+ATEST
 )%0A      
@@ -903,16 +903,13 @@
 on.L
-UCENE_42
+ATEST
 , Ch

Upgrade Lucene version