package de.nlptools.languageid.io;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * @author Daniil Sorokin <daniil.sorokin@uni-tuebingen.de>
 */
public class DocumentReader {
    
    public static final String ENCODING = "utf-8";
    
    /**
     * 
     * @param file
     * @return
     * @throws IOException 
     */
    public static String readContentFromFile(File file) throws IOException {
        String content = new String(Files.readAllBytes(file.toPath()), ENCODING);
        return content;
    }
    
    /**
     * Extracts an ISO language code from the file name. The file format should 
     * be in form LangIsoCode_DocumentID. The described format is used in 
     * the training and test data.
     * 
     * @param fileName The name of the file in question
     * @return an ISO language code
     */
    public static String getDocumentLanguageFromFileName(String fileName){
        return fileName.substring(0, fileName.indexOf('_'));
    }
    
    
    /**
     * 
     * @param dir
     * @return 
     */
    public static Dataset readDatasetFromFolder(String dir) {
        File dirFile = new File(dir);
        if (!dirFile.isDirectory()) return null;
        File[] files = dirFile.listFiles();
        ArrayList<String> documentsList = new ArrayList<>();
        ArrayList<String> labelsList = new ArrayList<>();
        for (File file : files) {
            try {
                String content = DocumentReader.readContentFromFile(file);
                documentsList.add(content);
                String lang = DocumentReader.getDocumentLanguageFromFileName(file.getName());
                labelsList.add(lang);
            } catch (IOException ex) {
                Logger.getLogger(DocumentReader.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        String[] documents = documentsList.toArray(new String[documentsList.size()]);
        String[] labels = labelsList.toArray(new String[labelsList.size()]);
        return new Dataset(documents, labels);
    }

    public static Dataset readDatasetFromMetaFile(String metaFileName) {
        File metaFile = new File(metaFileName);
        if (!metaFile.isFile()) return null;
        ArrayList<String> filesList = new ArrayList<>();        
        ArrayList<String> labelsList = new ArrayList<>();
        try (BufferedReader in = new BufferedReader(
                new InputStreamReader(
                new FileInputStream(metaFile), "UTF-8"))){
            String line;
            while((line = in.readLine()) != null){
                String[] columns = line.trim().split(",");
                filesList.add(columns[0]);
                labelsList.add(columns[1]);
            }
        } catch (IOException ex) {
            Logger.getLogger(DocumentReader.class.getName()).log(Level.SEVERE, null, ex);
        }
        ArrayList<String> documentsList = new ArrayList<>();
        for (String file : filesList) {
            try {
                String content = DocumentReader.readContentFromFile(new File(file));
                documentsList.add(content);
            } catch (IOException ex) {
                Logger.getLogger(DocumentReader.class.getName()).log(Level.SEVERE, null, ex);
            }
        }        
        String[] documents = documentsList.toArray(new String[documentsList.size()]);
        String[] labels = labelsList.toArray(new String[labelsList.size()]);
        return new Dataset(documents, labels);
    }    
}

@@ -797,16 +797,62 @@
 ODING);%0A
+        content = content.replace('%5Cn', ' ');%0A
         
@@ -867,16 +867,16 @@
 ontent;%0A
-
     %7D%0A  
@@ -1272,60 +1272,182 @@
     
-return fileName.substring(0, fileName.indexOf('_'));
+int index = fileName.indexOf('_');%0A        if (index != -1 && index %3C 5)%7B%0A            return fileName.substring(0, index);%0A        %7D else %7B%0A            return null;%0A        %7D
 %0A   
@@ -2101,32 +2101,50 @@
 %0A               
+ if (lang != null)
  labelsList.add(
@@ -2410,32 +2410,114 @@
 tring%5B%5D labels =
+ %7B%7D;%0A        if ( labelsList.size() == documentsList.size() )%0A            labels =
  labelsList.toAr
@@ -3220,24 +3220,68 @@
 olumns%5B0%5D);%0A
+                if (columns.length %3E 1)%0A    
             
@@ -3308,24 +3308,24 @@
 olumns%5B1%5D);%0A
-
             
@@ -3952,32 +3952,32 @@
 sList.size()%5D);%0A
-
         String%5B%5D
@@ -3977,32 +3977,114 @@
 tring%5B%5D labels =
+ %7B%7D;%0A        if ( labelsList.size() == documentsList.size() )%0A            labels =
  labelsList.toAr

Replace new line characters.