#!/usr/bin/env python
# -*- coding: utf-8 -*-
import MeCab

class Token:
    def __init__(self, surface, basic_form, part_of_speech_id):
        self.surface = surface
        self.basic_form = basic_form
        self.part_of_speech_id = part_of_speech_id

    def __str__(self):
        s = '%s-%s-%d' % (self.surface, self.basic_form, self.part_of_speech_id)
        return s.encode('utf-8')

def tokenize(text):
    mecab = MeCab.Tagger('-Ochasen')
    node = mecab.parseToNode(text)
    tokens = []
    while node:
        surface = node.surface.decode('utf-8')
        basic_form = node.feature.split(',')[6].decode('utf-8')
        if basic_form == '*':
            basic_form = surface
            
        token = Token(surface, basic_form, node.posid)
        tokens.append(token)
        
        node = node.next
        
    return tokens[1:-1]


if __name__ == '__main__':
    text = u'Emacs（イーマックス）とは高機能でカスタマイズ性の高いテキストエディタである。スクリーン・エディタとしての人気が高く、特にUNIXのプログラマを中心としたコンピュータ技術者に愛用者が多い。'

    tokens = tokenize(text.encode('utf-8'))

    for t in tokens:
        print t

@@ -56,314 +56,151 @@
 Cab%0A
-%0Aclass Token:%0A    def __init__(self, surface, basic_form, part_of_speech_id):%0A        self.surface = surface%0A        self.basic_form = basic_form%0A        self.part_of_speech_id = part_of_speech_id%0A%0A    def __str__(self):%0A        s = '%25s-%25s-%25d' %25 (self.surface, self.basic_form, self.part_of_speech_id
+import re%0A%0Adef tokenizeEn(text):%0A    tokens = text.split(' ')%0A    # remove symbols%0A    tokens = map(lambda t: re.sub(r'%5B%22,.%5D', '', t), tokens
 )%0A
-    
     
@@ -210,25 +210,14 @@
 urn 
-s.encode('utf-8')
+tokens
 %0A%0Ade
@@ -222,24 +222,26 @@
 def tokenize
+Jp
 (text):%0A    
@@ -526,105 +526,34 @@
     
-    %0A        token = Token(surface, basic_form, node.posid)%0A        tokens.append(token)%0A        
+tokens.append(basic_form)%0A
 %0A   
@@ -640,16 +640,19 @@
 _':%0A    
+jp_
 text = u
@@ -750,16 +750,287 @@
 %E7%94%A8%E8%80%85%E3%81%8C%E5%A4%9A%E3%81%84%E3%80%82'%0A
+    en_text = u'Emacs and its derivatives are a family of text editors that are characterized by their extensibility. The manual for the most widely-used variant, GNU Emacs, describes it as %22the extensible, customizable, self-documenting, real-time display editor%22.'%0A    
 %0A    tok
@@ -1043,17 +1043,22 @@
 tokenize
-(
+Jp(jp_
 text.enc
@@ -1075,42 +1075,108 @@
 '))%0A
-%0A
     
-for t in 
+print '%7C'.join(
 tokens
-:
+)%0A
 %0A    
-    print t
+tokens = tokenizeEn(en_text.encode('utf-8'))%0A    print '%7C'.join(tokens)
 %0A

Implement tokenizers, for Japanese and for English