package utilities;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;

import javax.swing.JOptionPane;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by Thagus on 05/09/16.
 */
public class TermExtractor {
    private static Pattern reg = Pattern.compile("[a-z]+");

    private static HashSet<String> stopWords;

    /**
     * Coordinates the initialization of the stopwords set and the dictionary for the spell checker
     */
    public static void initialize(){
        //Load stop words
        try {
            stopWords = new HashSet<>();
            readStopWords();
        } catch (IOException e) {
            e.printStackTrace();
            JOptionPane.showMessageDialog(null, e.toString(), "Error loading stopwords.txt", JOptionPane.ERROR_MESSAGE);
        }

    }

    /**
     * Read the stopwords file in order to feed the HashSet
     * @throws IOException in case that the file cant be opened or found
     */
    private static void readStopWords() throws IOException {
        LineIterator lineIterator = IOUtils.lineIterator(TermExtractor.class.getResourceAsStream("/stopwords.txt"), null);

        try {
            while (lineIterator.hasNext()) {
                String line = lineIterator.nextLine();
                stopWords.add(line.trim());
            }
            System.out.println(stopWords.size() + " stop words read");
        }
        finally {
            lineIterator.close();
        }
    }

    /**
     * Extracts the stemmed and corrected terms from a String
     * @param text The string from where obtain the terms
     * @return A HashMap containing as key the term and value its occurrence
     */
    public static HashMap<String, Integer> extractTerms(String text){
        HashMap<String, Integer> termResults = new HashMap<>();  //Will contain as key every term in the text, and value the frequency of the term
        ArrayList<String> words = new ArrayList<>();

        //Get only character strings from the text in lowercase
        Matcher matcher = reg.matcher(text.toLowerCase());

        while (matcher.find()){
            words.add(matcher.group()); //Add the matched strings to the words array
        }

        //Stem and remove stop words from the array
        words = stemmingAndStopWordsRemoval(words);

        //Count words and add them to a HashMap
        for(String word : words){
            Integer count = termResults.get(word);  //Check if the term is already on the HashMap and obtain its value
            //If it's not on the HashSet, initialize the count
            if(count==null){
                count = 0;
            }
            termResults.put(word, count+1);     //Add +1 for each term occurrence
        }

        //Return the termResults HashMap
        return termResults;
    }

    /**
     * A method to obtain stemmed words without stop words
     * @param words The original array containing words to stem
     * @return The new array containing stemmed words
     */
    private static ArrayList<String> stemmingAndStopWordsRemoval(ArrayList<String> words){
        ArrayList<String> stemmedWords = new ArrayList<>();
        for(String word : words){
            //Skip stop words
            if(!stopWords.contains(word)) {
                //Stem word
                String stemmed = PorterStemmer.stem(word);
                //Add the stemmed word to teh array
                stemmedWords.add(stemmed);
            }
        }
        return stemmedWords;
    }
}

@@ -235,16 +235,47 @@
 ashSet;%0A
+import java.util.ListIterator;%0A
 import j
@@ -459,24 +459,92 @@
 %22%5Ba-z%5D+%22);%0A%0A
+    private static boolean stopWordRemoval=true, useStemming=true;%0A%0A
     private 
@@ -2439,24 +2439,25 @@
      %7D%0A%0A
+%0A
         
 //Stem a
@@ -2452,103 +2452,239 @@
     
-//Stem and remove stop words from the array%0A        words = stemmingAndStopWordsRemoval(words);
+if(stopWordRemoval)%7B    //If we have to remove stop words, remove them%0A            removeStopWords(words);%0A        %7D%0A        if(useStemming)%7B    //If we should stem the word, stem them%0A            words = stemming(words);%0A        %7D
 %0A%0A  
@@ -3235,27 +3235,8 @@
 ords
- without stop words
 %0A   
@@ -3407,27 +3407,8 @@
 ming
-AndStopWordsRemoval
 (Arr
@@ -3543,22 +3543,16 @@
  //S
-kip stop
+tem
  word
-s
 %0A   
@@ -3564,219 +3564,643 @@
     
-if(!stopWords.contains(word)) %7B%0A                //Stem word%0A                String stemmed = PorterStemmer.stem(word);%0A                //Add the stemmed word to teh array%0A                stemmedWords.add(stemmed
+String stemmed = PorterStemmer.stem(word);%0A            //Add the stemmed word to teh array%0A            stemmedWords.add(stemmed);%0A        %7D%0A        return stemmedWords;%0A    %7D%0A%0A    /**%0A     * A method to remove stop words from an ArrayList%0A     * @param words the ArrayList that contains the words that will be checked%0A     */%0A    private static void removeStopWords(ArrayList%3CString%3E words)%7B%0A        ListIterator%3CString%3E iterator = words.listIterator();%0A        while (iterator.hasNext())%7B%0A            //Remove word if contained in the stopwords set%0A            if(stopWords.contains(iterator.next()))%7B%0A                iterator.remove(
 );%0A 
@@ -4226,39 +4226,260 @@
   %7D%0A    
+%7D%0A%0A
     
-return stemmedWords
+public static void setStopWordRemoval(boolean stopWordRemoval) %7B%0A        TermExtractor.stopWordRemoval = stopWordRemoval;%0A    %7D%0A%0A    public static void setUseStemming(boolean useStemming) %7B%0A        TermExtractor.useStemming = useStemming
 ;%0A    %7D%0A

Use of booleans to determine the use of stemming and stop word removal