from base import FieldType
from dedupe import predicates

from affinegap import normalizedAffineGapDistance
from simplecosine.cosine import CosineTextSimilarity, CosineSetSimilarity

class ShortStringType(FieldType) :
    comparator = normalizedAffineGapDistance
    type = "ShortString"

    _predicate_functions = (predicates.wholeFieldPredicate,
                            predicates.tokenFieldPredicate,
                            predicates.firstTokenPredicate,
                            predicates.commonIntegerPredicate,
                            predicates.nearIntegersPredicate,
                            predicates.firstIntegerPredicate,
                            predicates.sameThreeCharStartPredicate,
                            predicates.sameFiveCharStartPredicate,
                            predicates.sameSevenCharStartPredicate,
                            predicates.commonFourGram,
                            predicates.commonSixGram,
                            predicates.suffixArray,
                            predicates.commonTwoTokens,
                            predicates.commonThreeTokens,
                            predicates.fingerprint,
                            predicates.oneGramFingerprint,
                            predicates.twoGramFingerprint,
                            predicates.sortedAcronym,
                            predicates.doubleMetaphone,
                            predicates.metaphoneToken)

class StringType(ShortStringType) :
    comparator = normalizedAffineGapDistance
    type = "String"

    _canopy_thresholds = (0.2, 0.4, 0.6, 0.8)

    def __init__(self, definition) :
        super(StringType, self).__init__(definition)

        self.predicates += [predicates.TfidfTextPredicate(threshold, 
                                                          self.field)
                            for threshold in self._canopy_thresholds]

        self.predicates += [predicates.TfidfNGramPredicate(threshold, 
                                                          self.field)
                            for threshold in self._canopy_thresholds]



class TextType(StringType) :
    type = "Text"

    def __init__(self, definition) :
        super(TextType, self).__init__(definition)

        if 'corpus' not in definition :
            definition['corpus'] = []

        self.comparator = CosineTextSimilarity(definition['corpus'])

@@ -180,218 +180,52 @@
 ty%0A%0A
-class ShortStringType(FieldType) :%0A    comparator = normalizedAffineGapDistance%0A    type = %22ShortString%22%0A%0A    _predicate_functions = (predicates.wholeFieldPredicate,%0A                            predicates.token
+base_predicate_functions = (predicates.whole
 Fiel
@@ -690,169 +690,8 @@
 te,%0A
-                            predicates.commonFourGram,%0A                            predicates.commonSixGram,%0A                            predicates.suffixArray,%0A
     
@@ -1026,269 +1026,386 @@
 onym
-,%0A                            predicates.doubleMetaphone,%0A                            predicates.metaphoneToken)%0A%0Aclass StringType(ShortStringType) :%0A    comparator = normalizedAffineGapDistance%0A    type = %22String%22%0A%0A    _canopy_thresholds = (0.2, 0.4, 0.6, 0.8)
+)%0A%0Aclass IndexNGramString(object) :%0A%0A    def __init__(self, definition) :%0A        super(IndexNGramString, self).__init__(definition)%0A%0A        self.predicates += %5Bpredicates.TfidfNGramPredicate(threshold, %0A                                                          self.field)%0A                            for threshold in self._canopy_thresholds%5D%0A%0A%0Aclass IndexTextString(object) :
 %0A%0A  
@@ -1453,26 +1453,31 @@
   super(
+IndexText
 String
-Type
 , self).
@@ -1713,36 +1713,215 @@
 s%5D%0A%0A
-        self.predicate
+%0A%0A%0Aclass ShortStringType(IndexNGramString, FieldType) :%0A    comparator = normalizedAffineGapDistance%0A    type = %22ShortString%22%0A%0A    _canopy_thresholds = (0.2, 0.4, 0.6, 0.8)%0A%0A    _predicate_function
 s 
-+
 = 
-%5B
+(base_
 pred
@@ -1929,231 +1929,656 @@
 cate
-s.TfidfNGramPredicate(threshold, %0A                                                          self.field)%0A                            for threshold in self._canopy_thresholds%5D%0A%0A%0A%0Aclass TextType(StringType) :%0A    type = %22Text%22
+_functions %0A                            + (predicates.commonFourGram,%0A                               predicates.commonSixGram,%0A                               predicates.tokenFieldPredicate,%0A                               predicates.suffixArray,%0A                               predicates.doubleMetaphone,%0A                               predicates.metaphoneToken))%0A%0A%0Aclass StringType(IndexTextString, ShortStringType) :%0A    comparator = normalizedAffineGapDistance%0A    type = %22String%22%0A%0A%0A%0Aclass TextType(IndexTextString, FieldType) :%0A    type = %22Text%22%0A%0A    _predicate_functions = base_predicate_functions %0A    _canopy_thresholds = (0.2, 0.4, 0.6, 0.8)
 %0A%0A  
@@ -2656,32 +2656,33 @@
 __(definition)%0A%0A
+%0A
         if 'corp

refactor strings