package edu.umass.cs.jfoley.coop.conll;

import ciir.jfoley.chai.io.Directory;
import ciir.jfoley.chai.io.archive.ZipArchive;
import ciir.jfoley.chai.io.archive.ZipArchiveEntry;
import ciir.jfoley.chai.time.Debouncer;
import edu.umass.cs.jfoley.coop.coders.KryoCoder;
import edu.umass.cs.jfoley.coop.document.CoopDoc;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * @author jfoley
 */
public class CoopDocIndexer {
  public static void main(String[] args) throws IOException {
    Directory output = new Directory("clue_pre5.index");
    List<File> inputZips = new ArrayList<>();
    List<File> candidates = Directory.Read(".").children();
    for (File candidate : candidates) {
      if(candidate.getName().startsWith("clue_schemax")) {
        inputZips.add(candidate);
      }
    }
    KryoCoder<CoopDoc> coder = new KryoCoder<>(CoopDoc.class);

    Debouncer msg = new Debouncer(1000);

    long startTime = System.currentTimeMillis();
    try (TermBasedIndexWriter builder = new TermBasedIndexWriter(output)) {
      for (File inputZip : inputZips) {
        try (ZipArchive zip = ZipArchive.open(inputZip)) {
          List<ZipArchiveEntry> listEntries = zip.listEntries();
          for (int i = 0; i < listEntries.size(); i++) {
            ZipArchiveEntry entry = listEntries.get(i);
            CoopDoc doc = coder.read(entry.getInputStream());
            if(msg.ready()) {
              System.err.println(i + "/" + listEntries.size() + " " + doc.getName());
              System.err.println("# "+msg.estimate(i, listEntries.size()));
            }
            builder.addDocument(doc);
            if(i >= 50) break;
          }
        }
      }
      long endParsingTime = System.currentTimeMillis();
      System.out.println("Total parsing time: "+(endParsingTime - startTime)+"ms.");
    }
    long endTime = System.currentTimeMillis();
    System.out.println("Total time: "+(endTime - startTime)+"ms.");
    // 27.6s i>=50

  }
}

@@ -531,16 +531,103 @@
 ption %7B%0A
+    System.setProperty(%22java.util.concurrent.ForkJoinPool.common.parallelism%22, %2210%22);%0A%0A
     Dire
@@ -1764,16 +1764,17 @@
 (i %3E= 50
+0
 ) break;
@@ -2084,16 +2084,105 @@
 6s i%3E=50
+%0A    // 17.2s i%3E=50 with threaded Sorter%0A    // parse: 189,280ms, total: 239,167ms i%3E=500
 %0A%0A  %7D%0A%7D%0A

set forkjoin settings