from re import split

from spicedham.tokenizer import BaseTokenizer

class SplitTokenizer(BaseTokenizer):
    """
    Split the text on punctuation and newlines, lowercase everything, and
    filter the empty strings
    """
   
    def tokenize(self, text):
        text = split('[ ,.?!\n\r]', text)
        is_not_blank = lambda x: x != ''
        text = filter(is_not_blank, text)
        lower_case = lambda x: x.lower()
        text = map(lower_case, text)
        return text

@@ -306,160 +306,57 @@
     
-is_not_blank = lambda x: x != ''%0A        text = filter(is_not_blank, text)%0A        lower_case = lambda x: x.lower()%0A        text = map(lower_case, text)
+text = %5Btoken.lower() for token in text if token%5D
 %0A   

Make mapping & filtering into a list comprehension