package de.tblsoft.solr.pipeline;

import de.tblsoft.solr.util.IOUtils;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.Map;

/**
 * Created by tblsoft on 11.02.16.
 */
public class CSVReader extends AbstractReader {

    @Override
    public void read() {
        try {
            String filename = getProperty("filename", null);
            String delimiter = getProperty("delimiter", ",");
            String[] headers = getPropertyAsArray("headers", null);
            InputStream in = IOUtils.getInputStream(filename);
            java.io.Reader reader = new InputStreamReader(in);

            CSVFormat format = CSVFormat.RFC4180;
            if(headers == null) {
                format = format.withHeader();
            } else {
                format = format.withHeader(headers);
            }

            format=format.withDelimiter(delimiter.charAt(0));

            CSVParser parser = format.parse(reader);
            Iterator<CSVRecord> csvIterator = parser.iterator();

            while(csvIterator.hasNext()) {
                CSVRecord record = csvIterator.next();
                Map<String, Integer> header = parser.getHeaderMap();
                for(Map.Entry<String,Integer> entry : header.entrySet()) {
                    String key = entry.getKey();
                    String value = record.get(key);
                    executer.field(key, value);
                }
                executer.endDocument();
            }
            executer.end();
            in.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

}

@@ -39,38 +39,155 @@
 ort 
-de.tblsoft.solr.util.IOUtils
+java.io.InputStream;%0Aimport java.io.InputStreamReader;%0Aimport java.nio.charset.StandardCharsets;%0Aimport java.util.Iterator;%0Aimport java.util.Map
 ;%0A
+%0A
 impo
@@ -317,110 +317,36 @@
 ort 
-java.io.InputStream;%0Aimport java.io.InputStreamReader;%0Aimport java.util.Iterator;%0Aimport java.util.Map
+de.tblsoft.solr.util.IOUtils
 ;%0A%0A/
@@ -489,16 +489,107 @@
   try %7B%0A
+        %09%0A        %09String charset = getProperty(%22charset%22, StandardCharsets.UTF_8.name());%0A
         
@@ -830,24 +830,24 @@
 (filename);%0A
-
             
@@ -894,16 +894,24 @@
 eader(in
+,charset
 );%0A%0A    

add configuration for the charset to the csv reader