@@ -0,0 +1,3572 @@
+/*********************************************************************************%0A * Copyright (c) 2011, Monnet Project All rights reserved.%0A *%0A * Redistribution and use in source and binary forms, with or without%0A * modification, are permitted provided that the following conditions are met: *%0A * Redistributions of source code must retain the above copyright notice, this%0A * list of conditions and the following disclaimer. * Redistributions in binary%0A * form must reproduce the above copyright notice, this list of conditions and%0A * the following disclaimer in the documentation and/or other materials provided%0A * with the distribution. * Neither the name of the Monnet Project nor the names%0A * of its contributors may be used to endorse or promote products derived from%0A * this software without specific prior written permission.%0A *%0A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS %22AS IS%22%0A * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE%0A * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE%0A * ARE DISCLAIMED. IN NO EVENT SHALL THE MONNET PROJECT BE LIABLE FOR ANY%0A * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES%0A * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;%0A * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND%0A * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT%0A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS%0A * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.%0A * *******************************************************************************%0A */%0Apackage eu.monnetproject.translation.topics.experiments;%0A%0Aimport eu.monnetproject.translation.topics.CLIOpts;%0Aimport it.unimi.dsi.fastutil.ints.IntOpenHashSet;%0Aimport it.unimi.dsi.fastutil.ints.IntSet;%0Aimport java.io.DataInputStream;%0Aimport java.io.DataOutputStream;%0Aimport java.io.EOFException;%0Aimport java.io.File;%0A%0A/**%0A *%0A * @author John McCrae%0A */%0Apublic class FilterWordMap %7B%0A%0A    public static void main(String%5B%5D args) throws Exception %7B%0A        final CLIOpts opts = new CLIOpts(args);%0A        final File corpusFile = opts.roFile(%22corpus%5B.gz%7C.bz2%5D%22, %22The corpus to reduce the word map to%22);%0A        final File wordMapInFile = opts.roFile(%22wordMap%22, %22The full word map to reduce%22);%0A        final File wordMapOutFile = opts.woFile(%22out%22, %22The file to store the reduced wordMap%22);%0A        %0A        IntSet inCorpus = new IntOpenHashSet();%0A        %0A        final DataInputStream dis = new DataInputStream(CLIOpts.openInputAsMaybeZipped(corpusFile));%0A        %0A        while(dis.available() %3E 0) %7B%0A            try %7B%0A                int i = dis.readInt();%0A                inCorpus.add(i);%0A            %7D catch(EOFException x) %7B%0A                break;%0A            %7D%0A        %7D%0A        %0A        dis.close();%0A        %0A        final DataInputStream in = new DataInputStream(CLIOpts.openInputAsMaybeZipped(wordMapInFile));%0A        final DataOutputStream out = new DataOutputStream(CLIOpts.openOutputAsMaybeZipped(wordMapOutFile));%0A        %0A        while(in.available() %3E 0) %7B%0A            try %7B%0A                final String key = in.readUTF();%0A                final int i = in.readInt();%0A                if(inCorpus.contains(i)) %7B%0A                    out.writeUTF(key);%0A                    out.writeInt(i);%0A                %7D%0A            %7D catch(EOFException x) %7B%0A                break;%0A            %7D%0A        %7D%0A        in.close();%0A        out.flush();%0A        out.close();%0A        %0A    %7D%0A%7D%0A

Add wordmap filter for "bench-top" experiments