package parser;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.StreamTokenizer;
import java.util.Collection;

/**
 * @author Alexis Giraudet
 */
public class Parser {

    public static void parseLines(String fileName, Collection<String> c) throws IOException {
        BufferedReader bufferedReader = new BufferedReader(new FileReader(fileName));
        String line;
        while ((line = bufferedReader.readLine()) != null) {
            c.add(line.toLowerCase());
        }
    }

    public static void parseTokens(String fileName, Collection<String> c) throws IOException {
        StreamTokenizer streamTokenizer = new StreamTokenizer(new BufferedReader(new FileReader(fileName)));
        while(streamTokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if(streamTokenizer.ttype == StreamTokenizer.TT_WORD) {
                c.add(streamTokenizer.sval.toLowerCase());
            }
        }
    }
}

@@ -734,24 +734,624 @@
 ileName)));%0A
+        streamTokenizer.quoteChar('.');%0A        streamTokenizer.quoteChar(',');%0A        streamTokenizer.quoteChar(';');%0A        streamTokenizer.quoteChar(':');%0A        streamTokenizer.quoteChar('!');%0A        streamTokenizer.quoteChar(',');%0A        streamTokenizer.quoteChar('%22');%0A        streamTokenizer.quoteChar('(');%0A        streamTokenizer.quoteChar(')');%0A        streamTokenizer.quoteChar('%5B');%0A        streamTokenizer.quoteChar('%5D');%0A        streamTokenizer.quoteChar('%7B');%0A        streamTokenizer.quoteChar('%7D');%0A        streamTokenizer.quoteChar('%C2%AB');%0A        streamTokenizer.quoteChar('%C2%BB');%0A
         whil

add separator characters