package de.nlptools.languageid.io;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * @author Daniil Sorokin <daniil.sorokin@uni-tuebingen.de>
 */
public class DocumentReader {
    
    public static final String ENCODING = "utf-8";
    
    /**
     * 
     * @param file
     * @return
     * @throws IOException 
     */
    public static String readContentFromFile(File file) throws IOException {
        String content = new String(Files.readAllBytes(file.toPath()), ENCODING);
        return content;
    }
    
    /**
     * Extracts an ISO language code from the file name. The file format should 
     * be in form LangIsoCode_DocumentID. The described format is used in 
     * the training and test data.
     * 
     * @param fileName The name of the file in question
     * @return an ISO language code
     */
    public static String getDocumentLanguageFromFileName(String fileName){
        return fileName.substring(0, fileName.indexOf('_'));
    }
    
    
    /**
     * 
     * @param dir
     * @return 
     */
    public static Dataset readDatasetFromFolder(String dir) {
        File dirFile = new File(dir);
        if (!dirFile.isDirectory()) return null;
        File[] files = dirFile.listFiles();
        ArrayList<String> documentsList = new ArrayList<>();
        ArrayList<String> labelsList = new ArrayList<>();
        for (File file : files) {
            try {
                String content = DocumentReader.readContentFromFile(file);
                documentsList.add(content);
                String lang = DocumentReader.getDocumentLanguageFromFileName(file.getName());
                labelsList.add(lang);
            } catch (IOException ex) {
                Logger.getLogger(DocumentReader.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        String[] documents = documentsList.toArray(new String[documentsList.size()]);
        String[] labels = labelsList.toArray(new String[labelsList.size()]);
        return new Dataset(documents, labels);
    }
    
    
    public static void loadMetaFile(String file) throws IOException {
        try (BufferedReader in = new BufferedReader(
                new InputStreamReader(
                new FileInputStream(file), "UTF-8"))){
            TreeMap<String,Set<String>> fileDictionary = new TreeMap<>();
            String line;
            while((line = in.readLine()) != null){
                String[] columns = line.trim().split("\t");
                if(fileDictionary.containsKey(columns[2])) {
                    fileDictionary.get(columns[2]).add(columns[0]);
                } else {
                    HashSet<String> tmp = new HashSet<>();
                    tmp.add(columns[0]);
                    fileDictionary.put(columns[2], tmp);
                }
            }
        }
    }

    public static Map<String, String> loadListFromMetaFile(String file) {
        TreeMap<String,String> fileDictionary = new TreeMap<>();
        try (BufferedReader in = new BufferedReader(
                new InputStreamReader(
                new FileInputStream(file), "UTF-8"))){
            String line;
            while((line = in.readLine()) != null){
                String[] columns = line.trim().split("\t");
                fileDictionary.put(columns[0], columns[2]);
            }
        } catch (IOException ex) {
            Logger.getLogger(DocumentReader.class.getName()).log(Level.SEVERE, null, ex);
        }
        return fileDictionary;
    }    
    
}

@@ -2333,33 +2333,24 @@
 els);%0A    %7D%0A
-    %0A    
 %0A    public 
@@ -2360,910 +2360,273 @@
 tic 
-void loadMetaFile(String file) throws IOException %7B%0A        try (BufferedReader in = new BufferedReader(%0A                new InputStreamReader(%0A                new FileInputStream(file), %22UTF-8%22)))%7B%0A            TreeMap%3CString,Set%3CString%3E%3E fileDictionary = new TreeMap%3C%3E();%0A            String line;%0A            while((line = in.readLine()) != null)%7B%0A                String%5B%5D columns = line.trim().split(%22%5Ct%22);%0A                if(fileDictionary.containsKey(columns%5B2%5D)) %7B%0A                    fileDictionary.get(columns%5B2%5D).add(columns%5B0%5D);%0A                %7D else %7B%0A                    HashSet%3CString%3E tmp = new HashSet%3C%3E();%0A                    tmp.add(columns%5B0%5D);%0A                    fileDictionary.put(columns%5B2%5D, tmp);%0A                %7D%0A            %7D%0A        %7D%0A    %7D%0A%0A    public static Map%3CString, String%3E loadListFromMetaFile(String file) %7B%0A        TreeMap%3CString,String%3E fileDictionary = new TreeMap
+Dataset readDatasetFromMetaFile(String metaFileName) %7B%0A        File metaFile = new File(metaFileName);%0A        if (!metaFile.isFile()) return null;%0A        ArrayList%3CString%3E filesList = new ArrayList%3C%3E();        %0A        ArrayList%3CString%3E labelsList = new ArrayList
 %3C%3E()
@@ -2755,17 +2755,21 @@
 tStream(
-f
+metaF
 ile), %22U
@@ -2912,10 +2912,9 @@
 it(%22
-%5Ct
+,
 %22);%0A
@@ -2937,43 +2937,70 @@
 file
-Dictionary.put(columns%5B0%5D, 
+sList.add(columns%5B0%5D);%0A                labelsList.add(
 columns%5B
 2%5D);
@@ -2999,9 +2999,9 @@
 mns%5B
-2
+1
 %5D);%0A
@@ -3161,41 +3161,628 @@
     
-return fileDictionary
+ArrayList%3CString%3E documentsList = new ArrayList%3C%3E();%0A        for (String file : filesList) %7B%0A            try %7B%0A                String content = DocumentReader.readContentFromFile(new File(file));%0A                documentsList.add(content);%0A            %7D catch (IOException ex) %7B%0A                Logger.getLogger(DocumentReader.class.getName()).log(Level.SEVERE, null, ex);%0A            %7D%0A        %7D        %0A        String%5B%5D documents = documentsList.toArray(new String%5BdocumentsList.size()%5D);%0A        String%5B%5D labels = labelsList.toArray(new String%5BlabelsList.size()%5D)
 ;%0A    
-%7D    %0A
+    return new Dataset(documents, labels);%0A    %7D
     

Read from meta data.