/*
 * Copyright 2012-2015 Johns Hopkins University HLTCOE. All rights reserved.
 * See LICENSE in the project root directory.
 */
package annotatednyt;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.nytlabs.corpus.NYTCorpusDocument;
import com.nytlabs.corpus.NYTCorpusDocumentParser;

import edu.jhu.hlt.annotatednyt.AnnotatedNYTDocument;

/**
 *
 */
public class AnnotatedNYTConversionIT {
  
  private static final Logger LOGGER = LoggerFactory.getLogger(AnnotatedNYTConversionIT.class);
  
  Path nytData = Paths.get(System.getProperty("anytDataPath")).resolve("data");
  
  @Test
  public void corpusTest() throws IOException {
    final NYTCorpusDocumentParser parser = new NYTCorpusDocumentParser();
    Files.list(this.nytData)
        .flatMap(p -> {
          try {
            return Files.list(p); 
          } catch (IOException e) {
            throw new RuntimeException(e);
          }
        })
        .filter(p -> p.toString().endsWith(".tgz"))
        .forEach(p -> {
          NYTCorpusDocument nytd = parser.parseNYTCorpusDocumentFromFile(p.toFile(), true);
          LOGGER.info("Got NYT representation of document: {}", nytd.getGuid());
          AnnotatedNYTDocument anytd = new AnnotatedNYTDocument(nytd);
          LOGGER.info("ANYT representation: {}", anytd.toString());
        });
  }
}

@@ -161,27 +161,91 @@
 java.io.
-IOException
+BufferedInputStream;%0Aimport java.io.IOException;%0Aimport java.io.InputStream
 ;%0Aimport
@@ -500,16 +500,86 @@
 arser;%0A%0A
+import edu.jhu.hlt.acute.iterators.tar.TarGzArchiveEntryByteIterator;%0A
 import e
@@ -1287,81 +1287,369 @@
     
-NYTCorpusDocument nytd = parser.parseNYTCorpusDocumentFromFile(p.toFile()
+try (InputStream is = Files.newInputStream(p);%0A              BufferedInputStream bin = new BufferedInputStream(is, 1024 * 8 * 16);%0A              TarGzArchiveEntryByteIterator iter = new TarGzArchiveEntryByteIterator(bin);) %7B%0A            while (iter.hasNext()) %7B%0A              byte%5B%5D n = iter.next();%0A              NYTCorpusDocument nytd = parser.fromByteArray(n
 , tr
@@ -1655,32 +1655,36 @@
 rue);%0A          
+   
+ 
 LOGGER.info(%22Got
@@ -1748,16 +1748,20 @@
         
+    
 Annotate
@@ -1815,32 +1815,36 @@
 ytd);%0A          
+   
+ 
 LOGGER.info(%22ANY
@@ -1881,24 +1881,129 @@
 oString());%0A
+            %7D%0A          %7D catch (IOException e) %7B%0A            throw new RuntimeException(e);%0A          %7D%0A
         %7D);%0A

Update the IT to properly handle archives.