
package edu.ucla.library.ezid.crawler.utils;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;

public class InclusiveFilter {

    private static final File SOURCE = new File("/home/kevin/syriac-arks.csv");

    private static final File INCLUDES = new File("/home/kevin/syriac-includes.csv");

    private static final Logger LOGGER = LoggerFactory.getLogger(InclusiveFilter.class);

    public InclusiveFilter() {
    }

    public static void main(final String[] args) throws IOException {
        final List<String> patterns = new ArrayList<String>();
        final CSVReader iReader = new CSVReader(new FileReader(INCLUDES));
        final CSVReader sReader = new CSVReader(new FileReader(SOURCE));
        final CSVWriter writer = new CSVWriter(new FileWriter(new File("/home/kevin/syriac-filtered.csv")));

        String[] sourceData;
        String[] includes;

        while ((includes = iReader.readNext()) != null) {
            final String include = ".*\\/" + includes[0].replace("*", ".*");

            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("Adding an include pattern to the list of includes: {}", include);
            }

            patterns.add(include);
        }

        while ((sourceData = sReader.readNext()) != null) {
            for (int index = 0; index < patterns.size(); index++) {
                if (sourceData[1].matches(patterns.get(index))) {
                    if (LOGGER.isDebugEnabled()) {
                        LOGGER.debug("Adding file to the list to be ingested: {}", sourceData[0], sourceData[1]);
                    }

                    writer.writeNext(sourceData);
                }
            }
        }

        iReader.close();
        sReader.close();
        writer.close();
    }

}

@@ -44,29 +44,8 @@
 s;%0A%0A
-import java.io.File;%0A
 impo
@@ -63,24 +63,24 @@
 FileReader;%0A
+
 import java.
@@ -232,16 +232,59 @@
 ctory;%0A%0A
+import info.freelibrary.util.StringUtils;%0A%0A
 import a
@@ -396,176 +396,8 @@
  %7B%0A%0A
-    private static final File SOURCE = new File(%22/home/kevin/syriac-arks.csv%22);%0A%0A    private static final File INCLUDES = new File(%22/home/kevin/syriac-includes.csv%22);%0A%0A
     
@@ -720,16 +720,54 @@
 der(
-INCLUDES
+StringUtils.format(%22%7B%7D_includes.csv%22, args%5B0%5D)
 ));%0A
@@ -834,13 +834,49 @@
 er(S
-OURCE
+tringUtils.format(%22%7B%7D_ezid.csv%22, args%5B0%5D)
 ));%0A
@@ -941,37 +941,31 @@
 ter(
-new File(%22/home/kevin/syriac-
+StringUtils.format(%22%7B%7D_
 filt
@@ -973,16 +973,25 @@
 red.csv%22
+, args%5B0%5D
 )));%0A%0A  

Update InclusiveFilter to be more generic