package com.metaweb.gridworks.expr.functions.strings;

import java.util.Arrays;
import java.util.Properties;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.json.JSONException;
import org.json.JSONWriter;

import com.metaweb.gridworks.gel.Function;

public class Fingerprint implements Function {

    static final Pattern alphanum = Pattern.compile("\\p{Punct}|\\p{Cntrl}");
    
    public Object call(Properties bindings, Object[] args) {
        if (args.length == 1 && args[0] != null) {
            Object o = args[0];
            String s = (o instanceof String) ? (String) o : o.toString(); 
            s = s.trim(); // first off, remove whitespace around the string
            s = s.toLowerCase(); // then lowercase it
            s = alphanum.matcher(s).replaceAll(""); // then remove all punctuation and control chars
            String[] frags = StringUtils.split(s); // split by whitespace
            Arrays.sort(frags); // sort the fragments
            return StringUtils.join(frags," "); // rejoin them with a single space between them 
            
        }
        return null;
    }
    
    public void write(JSONWriter writer, Properties options)
        throws JSONException {
    
        writer.object();
        writer.key("description"); writer.value("Returns the fingerprint of s, a derived string that aims to be a more canonical form of it (this is mostly useful for finding clusters of strings related to the same information).");
        writer.key("params"); writer.value("string s");
        writer.key("returns"); writer.value("string");
        writer.endObject();
    }
}

@@ -65,22 +65,24 @@
 va.util.
-Arrays
+Iterator
 ;%0Aimport
@@ -104,16 +104,42 @@
 erties;%0A
+import java.util.TreeSet;%0A
 import j
@@ -982,39 +982,128 @@
     
-Arrays.sort(frag
+TreeSet%3CString%3E set = new TreeSet%3CString%3E();%0A            for (String ss : frags) %7B%0A                set.add(s
 s); // 
-s
 or
-t the
+der
  fra
@@ -1108,16 +1108,27 @@
 ragments
+ and dedupe
 %0A       
@@ -1136,105 +1136,295 @@
     
-return StringUtils.join(frags,%22 %22); // rejoin them with a single space between them %0A            
+%7D%0A            StringBuffer b = new StringBuffer();%0A            Iterator%3CString%3E i = set.iterator();%0A            while (i.hasNext()) %7B%0A                b.append(i.next());%0A                b.append(' ');%0A            %7D%0A            return b.toString(); // join ordered fragments back together
 %0A   

use a TreeSet to do both sorting and de-dupe of the split fragments