@@ -0,0 +1,1959 @@
+package jfoley.vocabpress;%0A%0Aimport ciir.jfoley.chai.io.IO;%0Aimport ciir.jfoley.chai.string.StrUtil;%0Aimport org.lemurproject.galago.core.parse.Document;%0Aimport org.lemurproject.galago.core.parse.TagTokenizer;%0Aimport org.lemurproject.galago.core.parse.TrecTextParser;%0Aimport org.lemurproject.galago.core.util.DocumentSplitFactory;%0Aimport org.lemurproject.galago.tupleflow.Utility;%0Aimport org.lemurproject.galago.utility.Parameters;%0Aimport org.lemurproject.galago.utility.tools.Arguments;%0A%0Aimport java.io.File;%0Aimport java.io.IOException;%0Aimport java.io.PrintWriter;%0Aimport java.util.Arrays;%0Aimport java.util.List;%0A%0A/**%0A * @author jfoley%0A */%0Apublic class ConvertToSimple %7B%0A  public static void convert(String input, String output, Parameters argp) throws IOException %7B%0A    TagTokenizer tok = new TagTokenizer();%0A    try (%0A        TrecTextParser parser = new TrecTextParser(DocumentSplitFactory.file(input), argp);%0A        PrintWriter out = IO.openPrintWriter(output)%0A    ) %7B%0A      while(true) %7B%0A        Document doc = parser.nextDocument();%0A        if(doc == null) break;%0A        System.err.println(%22# %22+doc.name);%0A        tok.tokenize(doc);%0A        out.println(Parameters.parseArray(%22text%22, Utility.join(doc.terms, %22 %22), %22id%22, doc.name));%0A      %7D%0A    %7D%0A%0A    System.err.println(%22Done!%22);%0A  %7D%0A%0A  public static void main(String%5B%5D args) throws IOException %7B%0A    Parameters argp = Arguments.parse(args);%0A%0A    String basePath = %22/mnt/scratch/jfoley/robust04raw/%22;%0A    List%3CString%3E pieces = Arrays.asList(%0A        %22ft91.dat.gz%22, %22ft92.dat.gz%22, %22ft93.dat.gz%22, %22ft94.dat.gz%22,%0A        %22fr94.dat.gz%22, %22fbis5.dat.gz%22, %22latimes.dat.gz%22);%0A%0A    for (String piece : pieces) %7B%0A      String input = basePath + piece;%0A      String output = StrUtil.removeBack(piece, %22.dat.gz%22) + %22.json.gz%22;%0A      if(new File(output).exists()) %7B%0A        continue;%0A      %7D%0A      convert(input, output, argp);%0A      System.err.println(%22Done: %22 + piece);%0A    %7D%0A%0A    System.err.println(%22Done!%22);%0A  %7D%0A%7D%0A

Convert TrecText to simple json gz