@@ -0,0 +1,2195 @@
+package de.tblsoft.solr.pipeline.filter;%0A%0Aimport de.tblsoft.solr.pipeline.AbstractFilter;%0Aimport de.tblsoft.solr.pipeline.bean.Document;%0Aimport de.tblsoft.solr.util.IOUtils;%0Aimport org.apache.commons.io.FileUtils;%0A%0Aimport java.io.File;%0Aimport java.io.IOException;%0Aimport java.util.HashSet;%0Aimport java.util.Set;%0A%0A/**%0A * Created by tbl on 12.08.17.%0A */%0Apublic class CheckDuplicateFilter extends AbstractFilter %7B%0A%0A    private Set%3CString%3E duplicates;%0A%0A    private File absoluteDuplicatesStoreFile;%0A    private String duplicateFieldName;%0A%0A    private int counter = 0;%0A%0A    private int checkpoint = 1000;%0A%0A    @Override%0A    public void init() %7B%0A        String duplicatesStoreFile = getProperty(%22duplicatesStoreFile%22, %22duplicatesStore.txt%22);%0A        duplicateFieldName = getProperty(%22duplicateFieldName%22, null);%0A        checkpoint = getPropertyAsInt(%22checkpoint%22, 1000);%0A        absoluteDuplicatesStoreFile =  IOUtils.getAbsoluteFileAsFile(getBaseDir(),duplicatesStoreFile );%0A        if(!absoluteDuplicatesStoreFile.exists()) %7B%0A            duplicates = new HashSet%3CString%3E();%0A        %7D else %7B%0A            try %7B%0A                for(String line :FileUtils.readLines(absoluteDuplicatesStoreFile)) %7B%0A                    duplicates.add(line);%0A                %7D%0A            %7D catch (IOException e) %7B%0A                duplicates = new HashSet%3CString%3E();%0A            %7D%0A        %7D%0A        super.init();%0A    %7D%0A%0A    @Override%0A    public void document(Document document) %7B%0A        counter++;%0A        if((counter %25 checkpoint) == 0) %7B%0A            saveDuplicatesToFile();%0A        %7D%0A        String duplicateFieldValue = document.getFieldValue(duplicateFieldName);%0A        if(duplicateFieldValue == null) %7B%0A            super.document(document);%0A        %7D%0A        if(duplicates.contains(duplicateFieldValue)) %7B%0A            // do nothing%0A        %7D else %7B%0A            duplicates.add(duplicateFieldValue);%0A        %7D%0A    %7D%0A%0A    @Override%0A    public void end() %7B%0A        saveDuplicatesToFile();%0A%0A        super.end();%0A    %7D%0A%0A    void saveDuplicatesToFile() %7B%0A        try %7B%0A            FileUtils.writeLines(absoluteDuplicatesStoreFile, duplicates);%0A        %7D catch (IOException e) %7B%0A            //fail silent%0A        %7D%0A    %7D%0A%7D%0A

implement a check duplicate filter