@@ -0,0 +1,1816 @@
+package finalir;%0D%0A%0D%0Aimport edu.stanford.nlp.ling.CoreAnnotations;%0D%0Aimport edu.stanford.nlp.ling.CoreLabel;%0D%0Aimport edu.stanford.nlp.pipeline.Annotation;%0D%0Aimport edu.stanford.nlp.pipeline.StanfordCoreNLP;%0D%0Aimport java.io.*;%0D%0Aimport java.nio.file.Files;%0D%0Aimport java.util.*;%0D%0A%0D%0Apublic class Tokenizer %7B%0D%0A    %0D%0A    Properties props = new Properties();%0D%0A    StanfordCoreNLP pipeline;%0D%0A%0D%0A    HashSet%3CString%3E stopwords = new HashSet%3C%3E();%0D%0A    %0D%0A    public Tokenizer() %7B%0D%0A%0D%0A        props.setProperty(%22annotators%22, %22tokenize,ssplit,pos,lemma%22);%0D%0A        pipeline = new StanfordCoreNLP(props);%0D%0A        %0D%0A        try%0D%0A        %7B%0D%0A            stopwords.addAll(Files.readAllLines(new File(%22src%5C%5Cfinalir%5C%5Cstopwords.txt%22).toPath()));%0D%0A        %7D%0D%0A        catch(IOException ex)%7B%0D%0A            IR.PrintErr(%22Can't load stopwords file . . .%22);%0D%0A        %7D%0D%0A    %7D%0D%0A    %0D%0A    public List%3CCoreLabel%3E getTokens(File f) throws FileNotFoundException, IOException %7B%0D%0A        return getTokens(new Scanner(f).useDelimiter(%22%5C%5CZ%22).next());%0D%0A    %7D%0D%0A%0D%0A    public List%3CCoreLabel%3E getTokens(String queri)%7B%0D%0A        Annotation ann = new Annotation(queri);%0D%0A        pipeline.annotate(ann);%0D%0A        return removeStopWords(ann.get(CoreAnnotations.TokensAnnotation.class));%0D%0A    %7D%0D%0A%0D%0A    private int removedWordsCount = 0;%0D%0A    public int getRemovedWordsCount()%7B%0D%0A        return removedWordsCount;%0D%0A    %7D%0D%0A    %0D%0A    private List%3CCoreLabel%3E removeStopWords(List%3CCoreLabel%3E tokens) %7B%0D%0A        int tmp =  tokens.size();%0D%0A        %0D%0A        tokens.removeIf( x -%3E stopwords.contains(x.word()) );%0D%0A        %0D%0A//        for (int i = 0; i %3C tokens.size(); i++)%0D%0A//            if (stopwords.contains(tokens.get(i).originalText()))%0D%0A//                tokens.remove(i--);%0D%0A         %0D%0A        removedWordsCount += tmp - tokens.size();%0D%0A        return tokens;%0D%0A    %7D%0D%0A%7D%0D%0A

fix stopwords remover