package org.oscii.corpus;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Stream;

/**
 * Indexed corpus.
 */
public class Corpus {
    final Map<String, PostingList> postings = new HashMap<>();
    final List<String[]> lines = new ArrayList<>();
    final Tokenizer tokenizer;
    private final static Logger log = LogManager.getLogger(Corpus.class);

    public Corpus(Tokenizer tokenizer) {
        this.tokenizer = tokenizer;
    }

    public void addLines(String path) throws IOException {
        log.info("Loading corpus from {}", path);
        Stream<String> newLines = Files.lines(Paths.get(path)).parallel();
        addLines(newLines);
    }

    public void addLines(Stream<String> newLines) {
        int start = lines.size();
        newLines.map(tokenizer::tokenize).map(this::internAll).forEachOrdered(lines::add);
        for (int i = start; i < lines.size(); i++) {
            String[] tokens = lines.get(i);
            for (int j = 0; j < tokens.length; j++) {
                indexWord(tokens[j], i, j);
            }
        }
    }

    public int count(String word) {
        return postings(word).size();
    }

    public PostingList postings(String word) {
        return postings.getOrDefault(word, PostingList.EMPTY);
    }

    private String[] internAll(String[] strings) {
        for (int i = 0; i < strings.length; i++) {
            strings[i] = strings[i].intern();
        }
        return strings;
    }

    private void indexWord(String word, int line, int position) {
        PostingList p = postings.getOrDefault(word, null);
        if (p == null) {
            p = new PostingList();
            postings.put(word, p);
        }
        p.addPosting(line, position);
    }

    public Set<String> vocab() {
        return postings.keySet();
    }
}

@@ -120,16 +120,208 @@
 java.io.
+BufferedInputStream;%0Aimport java.io.BufferedReader;%0Aimport java.io.Closeable;%0Aimport java.io.IOException;%0Aimport java.io.InputStream;%0Aimport java.io.InputStreamReader;%0Aimport java.io.Unchecked
 IOExcept
@@ -317,32 +317,32 @@
 kedIOException;%0A
-
 import java.nio.
@@ -432,16 +432,54 @@
 .Stream;
+%0Aimport java.util.zip.GZIPInputStream;
 %0A%0A/**%0A *
@@ -949,32 +949,1111 @@
   Stream%3CString%3E
+ newLines;%0A        if (path.endsWith(%22.gz%22)) %7B%0A            // From https://erikwramner.wordpress.com/2014/05/02/lazily-read-lines-from-gzip-file-with-java-8-streams/%0A            InputStream fileIs = null;%0A            BufferedInputStream bufferedIs = null;%0A            GZIPInputStream gzipIs = null;%0A            try %7B%0A                fileIs = Files.newInputStream(Paths.get(path));%0A                // Even though GZIPInputStream has a buffer it reads individual bytes%0A                // when processing the header, better add a buffer in-between%0A                bufferedIs = new BufferedInputStream(fileIs, 65535);%0A                gzipIs = new GZIPInputStream(bufferedIs);%0A            %7D catch (IOException e) %7B%0A                closeSafely(gzipIs);%0A                closeSafely(bufferedIs);%0A                closeSafely(fileIs);%0A                throw new UncheckedIOException(e);%0A            %7D%0A            BufferedReader reader = new BufferedReader(new InputStreamReader(gzipIs));%0A            newLines = reader.lines().onClose(() -%3E closeSafely(reader));%0A        %7D else %7B%0A           
  newLines = File
@@ -2085,24 +2085,34 @@
 parallel();%0A
+        %7D%0A
         addL
@@ -2126,32 +2126,272 @@
 wLines);%0A    %7D%0A%0A
+    private static void closeSafely(Closeable closeable) %7B%0A        if (closeable != null) %7B%0A            try %7B%0A                closeable.close();%0A            %7D catch (IOException e) %7B%0A                // Ignore%0A            %7D%0A        %7D%0A    %7D%0A%0A
     public void 

allow gzipped corpus