import Stemmer
import string

#Tokenise the documents                 
class PorterTokeniser(object):
    def __init__(self):
        self.stemmer = Stemmer.Stemmer('english')
        self.minWordLength = 2
     
    def __call__(self, doc):
        doc = str(doc.lower())
        doc = doc.translate(string.maketrans("",""), string.punctuation)
        tokens =  [self.stemmer.stemWord(t) for t in doc.split()]  
        return [token for token in tokens if len(token) >= self.minWordLength]
@@ -253,12 +253,8 @@
 c = 
-str(
 doc.
@@ -260,16 +260,39 @@
 .lower()
+.encode('utf-8').strip(
 )%0A      
@@ -357,16 +357,32 @@
 tuation)
+.decode(%22utf-8%22)
 %0A       

Fix for unicode error