package jfoley.vocabpress;

import ciir.jfoley.chai.io.LinesIterable;
import gnu.trove.list.array.TIntArrayList;
import org.lemurproject.galago.core.btree.simple.DiskMapReader;
import org.lemurproject.galago.utility.Parameters;
import org.lemurproject.galago.utility.compression.VByte;

import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;

/**
 * @author jfoley
 */
public class CompressDocuments {

  public static byte[] termsAsVbytes(TIntArrayList terms) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);

    for (int i = 0; i < terms.size(); i++) {
      VByte.compressInt(dos, terms.get(i));
    }

    return baos.toByteArray();
  }

  public static void main(String[] args) throws IOException {
    Vocabulary vocab = new Vocabulary(new DiskMapReader("vocab.mapping.btree"));


    File inputDir = new File("/home/jfoley/code/synthesis/robust04simple");
    File[] files = inputDir.listFiles();
    if(files == null) throw new RuntimeException("can't list files!");

    for (File file : files) {
      int iter = 0;
      for (String line : LinesIterable.fromFile(file)) {
        if(++iter > 30) break;
        Parameters doc = Parameters.parseString(line);
        String text = doc.getString("text");
        TIntArrayList docAsTerms = new TIntArrayList();
        for (String term : text.split(" ")) {
          docAsTerms.add(vocab.getTermId(term));
        }

        byte[] vbyte = termsAsVbytes(docAsTerms);

        System.out.printf("%d\t%d\t%d\t%d\n", docAsTerms.size(), text.length(), docAsTerms.size()*4, vbyte.length);
      }
    }
  }
}

@@ -106,16 +106,126 @@
 ayList;%0A
+import gnu.trove.map.hash.TObjectIntHashMap;%0Aimport org.lemurproject.galago.core.btree.simple.DiskMapBuilder;%0A
 import o
@@ -509,16 +509,58 @@
 ception;
+%0Aimport java.nio.charset.StandardCharsets;
 %0A%0A/**%0A *
@@ -942,16 +942,333 @@
 );%0A  %7D%0A%0A
+  public static byte%5B%5D termsAsInts(TIntArrayList terms) throws IOException %7B%0A    ByteArrayOutputStream baos = new ByteArrayOutputStream();%0A    DataOutputStream dos = new DataOutputStream(baos);%0A%0A    for (int i = 0; i %3C terms.size(); i++) %7B%0A      dos.writeInt(terms.get(i));%0A    %7D%0A%0A    return baos.toByteArray();%0A  %7D%0A%0A
   public
@@ -1601,30 +1601,226 @@
     
-for (File file : files
+TObjectIntHashMap%3CString%3E docIdCache = new TObjectIntHashMap%3C%3E();%0A    try (%0A        DiskMapBuilder dmbvbyte = new DiskMapBuilder(%22corpus.vbyte%22);%0A        DiskMapBuilder dmbint4 = new DiskMapBuilder(%22corpus.int4%22);%0A    
 ) %7B%0A
@@ -1839,16 +1839,50 @@
 er = 0;%0A
+      for (File file : files) %7B%0A  
       fo
@@ -1938,16 +1938,18 @@
         
+  
 if(++ite
@@ -1954,21 +1954,114 @@
 ter 
-%3E 30) break;%0A
+%25 1000 == 0) %7B%0A            System.err.println(%22Docs: %22+iter+ %22 cache: %22+docIdCache.size());%0A          %7D%0A  
     
@@ -2119,16 +2119,18 @@
         
+  
 String t
@@ -2158,24 +2158,25 @@
 text%22);%0A
+%0A
         
 TIntArra
@@ -2167,16 +2167,18 @@
         
+  
 TIntArra
@@ -2217,32 +2217,34 @@
 List();%0A        
+ 
+ 
 for (String term
@@ -2279,55 +2279,280 @@
     
-docAsTerms.add(vocab.getTermId(term));%0A
+  int idterm = -1;%0A            if(docIdCache.containsKey(term)) %7B%0A              idterm = docIdCache.get(term);%0A            %7D else %7B%0A              idterm = vocab.getTermId(term);%0A              if(idterm %3C 50000) %7B%0A                docIdCache.put(term, idterm);%0A     
         
 %7D%0A%0A 
@@ -2551,70 +2551,312 @@
     
+ 
 %7D%0A
-%0A        byte%5B%5D vbyte = termsAsVbytes(docAsTerms);%0A%0A        
+            %7D%0A            docAsTerms.add(idterm);%0A          %7D%0A%0A          byte%5B%5D docId = doc.getString(%22id%22).getBytes(StandardCharsets.UTF_8);%0A%0A          byte%5B%5D vbyte = termsAsVbytes(docAsTerms);%0A          dmbvbyte.put(docId, vbyte);%0A          dmbint4.put(docId, termsAsInts(docAsTerms));%0A%0A          //
 Syst
@@ -2940,17 +2940,19 @@
 s.size()
-*
+ * 
 4, vbyte
@@ -2961,16 +2961,26 @@
 ength);%0A
+        %7D%0A
       %7D%0A

Modify compress docs