/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package org.mskcc.cbio.oncokb.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;

/**
 *
 * @author jgao
 */
public class FileUtils {
    private FileUtils() {
        throw new AssertionError();
    }
    
    public static List<String> getFilesInFolder(final String pathToFolder, final String suffix) {
        File folder = new File(pathToFolder);
        
        String[] files = folder.list(new FilenameFilter() {
            public boolean accept(File dir, String name) {
                return name.toUpperCase().endsWith(suffix.toUpperCase());
            }
        });
        
        List<String> ret = new ArrayList<String>();
        for (String file : files) {
            ret.add(pathToFolder + File.separator + file);
        }
        
        return ret;
    }
    
    /**
     * read local files and return content
     * @param pathToFile
     * @return
     * @throws IOException 
     */
    public static String readLocal(String pathToFile) throws IOException {
        return readStream(new FileInputStream(pathToFile));
    }
    
    /**
     * return remote files and return content
     * @param urlToFile
     * @return
     * @throws IOException 
     */
    public static String readRemote(String urlToFile) throws IOException {
        URL url = new URL(urlToFile);
        return readStream(url.openStream());
    }
    
    /**
     * read a stream and return content
     * @param is
     * @return
     * @throws IOException 
     */
    public static String readStream(InputStream is) throws IOException {
        List<String> lines = readTrimedLinesStream(is);
        return StringUtils.join(lines, "\n");
    }
    
    public static List<String> readTrimedLinesStream(InputStream is) throws IOException {
        return readLinesStream(is, true);
    }
    
    /**
     * read a stream and return lines
     * @param is
     * @return
     * @throws IOException 
     */
    public static List<String> readLinesStream(InputStream is, boolean trim) throws IOException {
        BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"));

        List<String> lines = new ArrayList<String>();
        String line;
        while ((line = in.readLine()) != null) {
            if (trim) {
                line = line.replaceAll("^[\uFEFF-\uFFFF]+", ""); // trim and remove unicode
                line = line.replaceAll("\\[[a-z]\\]", ""); // remove comments from google docs
                line = line.trim();
            }
            if (!line.isEmpty())
                lines.add(line);
        }
        in.close();
        
        return lines;
    }
    
    public static List<String> readLinesDocStream(InputStream is, boolean trim) throws IOException {
        XWPFDocument document=new XWPFDocument(is);
        List<XWPFParagraph> paragraphs = document.getParagraphs();
        List<String> ret = new ArrayList<String>();
        for(XWPFParagraph paragraph : paragraphs){
            String[] lines = paragraph.getParagraphText().split("\n");
            for (String line : lines) {
                if (trim) {
                    line = line.replaceAll("^[\uFEFF-\uFFFF]+", ""); // trim and remove unicode
                    line = line.replaceAll("\\[[a-z]\\]", ""); // remove comments from google docs
                    line = line.trim();
                }
                if (!line.isEmpty()) {
                    ret.add(line);
                }
            }
        }
        return ret;
    }
}

@@ -2752,1204 +2752,155 @@
 ine.
-replaceAll(%22%5E%5B%5CuFEFF-%5CuFFFF%5D+%22, %22%22); // trim and remove unicode%0A                line = line.replaceAll(%22%5C%5C%5B%5Ba-z%5D%5C%5C%5D%22, %22%22); // remove comments from google docs%0A                line = line.trim();%0A            %7D%0A            if (!line.isEmpty())%0A                lines.add(line);%0A        %7D%0A        in.close();%0A        %0A        return lines;%0A    %7D%0A    %0A    public static List%3CString%3E readLinesDocStream(InputStream is, boolean trim) throws IOException %7B%0A        XWPFDocument document=new XWPFDocument(is);%0A        List%3CXWPFParagraph%3E paragraphs = document.getParagraphs();%0A        List%3CString%3E ret = new ArrayList%3CString%3E();%0A        for(XWPFParagraph paragraph : paragraphs)%7B%0A            String%5B%5D lines = paragraph.getParagraphText().split(%22%5Cn%22);%0A            for (String line : lines) %7B%0A                if (trim) %7B%0A                    line = line.replaceAll(%22%5E%5B%5CuFEFF-%5CuFFFF%5D+%22, %22%22); // trim and remove unicode%0A                    line = line.replaceAll(%22%5C%5C%5B%5Ba-z%5D%5C%5C%5D%22, %22%22); // remove comments from google docs%0A                    line = line.trim();%0A                %7D%0A                if (!line.isEmpty()) %7B%0A                    ret.add(line);%0A                %7D%0A            %7D%0A        %7D%0A        return ret
+trim();%0A            %7D%0A            if (!line.isEmpty())%0A                lines.add(line);%0A        %7D%0A        in.close();%0A        %0A        return lines
 ;%0A  

Remove unnecessary replacement