package de.tblsoft.solr.pipeline.filter;

import de.tblsoft.solr.pipeline.AbstractFilter;
import de.tblsoft.solr.pipeline.bean.Document;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

/**
 * Created by tblsoft on 03.04.16.
 * 
 * Count the tokens for a specific field.
 */
public class TokenCounterFilter extends AbstractFilter {
	
	private String fieldName;
	private String tokenCountFieldName;

    @Override
    public void init() {
        fieldName = getProperty("fieldName", null);
        verify(fieldName, "For the TokenCounterFilter a fieldName property must be defined.");

        tokenCountFieldName = getProperty("tokenCountFieldName", "tokenCount");
        super.init();
    }


    @Override
    public void document(Document document) {
        List<String> values = document.getFieldValues(fieldName, new ArrayList<String>());
        int tokenCount = 0;
        for(String value: values) {
            StringTokenizer tokenizer = new StringTokenizer(value);
            tokenCount = tokenCount + tokenizer.countTokens();
        }
        document.setField(tokenCountFieldName, String.valueOf(tokenCount));


        super.document(document);
    }

}

@@ -46,49 +46,87 @@
 ort 
-de.tblsoft.solr.pipeline.AbstractFilt
+java.util.ArrayList;%0Aimport java.util.List;%0Aimport java.util.StringTokeniz
 er;%0A
+%0A
 impo
@@ -157,107 +157,69 @@
 ine.
-bean.Document;%0A%0Aimport java.util.ArrayList;%0Aimport java.util.List;%0Aimport java.util.StringTokenizer
+AbstractFilter;%0Aimport de.tblsoft.solr.pipeline.bean.Document
 ;%0A%0A/
@@ -250,13 +250,13 @@
  on 
-0
 3
+0
 .0
-4
+6
 .16.
@@ -273,21 +273,25 @@
 unt the 
-token
+character
 s for a 
@@ -1205,11 +1205,15 @@
 ;%0A    %7D%0A
+    
 %0A%7D%0A

improve the token count filter