package rivet.extras.text;

import static java.util.Arrays.stream;

import rivet.core.arraylabels.*;
import rivet.core.labels.ArrayRIV;
import rivet.core.labels.RandomIndexVector;

public final class UntrainedWords {
    private UntrainedWords(){}
    
    public static String[] tokenizeText (String text) {
        return text.split("\\s+");
    }
    
    public static ArrayRIV[] rivWords (String[] words, int size, int k) {
        ArrayRIV[] res = new ArrayRIV[words.length];
        for (int i = 0; i < words.length; i++) {
            ArrayRIV riv = ArrayRIV.generateLabel(size, k, words[i]);
            res[i] = riv;
        }
        return res;
    }
    
    public static ArrayRIV sumArrayRIVs (ArrayRIV[] rivs) { 
        return stream(rivs)
                .reduce(
                        new ArrayRIV(rivs[0].size()),
                        (i, r) -> i.add(r));
    }
    
    public static ArrayRIV rivettizeText (String text, int size, int k) {
        return sumArrayRIVs(rivWords(tokenizeText(text), size, k));
    }
    
}

@@ -83,100 +83,23 @@
 ore.
-arraylabels.*;%0Aimport rivet.core.labels.ArrayRIV;%0Aimport rivet.core.labels.RandomIndexVector
+labels.ArrayRIV
 ;%0A%0Ap

remove bogus dependencies