import re
import math
import os,sys
import pprint
import finalseg
import time

FREQ = {}
total =0.0

def gen_trie(f_name):
	global total
	trie = {}
	content = open(f_name,'rb').read().decode('utf-8')
	for line in content.split("\n"):
		word,freq = line.split(" ")
		freq = float(freq)
		FREQ[word] = freq
		total+=freq
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
	return trie


_curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

print >> sys.stderr, "Building Trie..."
t1 = time.time()
trie = gen_trie(os.path.join(_curpath,"dict.txt"))
FREQ = dict([(k,float(v)/total) for k,v in FREQ.iteritems()]) #normalize
min_freq = min(FREQ.itervalues())
print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds."
print >> sys.stderr, "Trie has been built succesfully."


def __cut_all(sentence):
	N = len(sentence)
	i,j=0,0
	p = trie
	while i<N:
		c = sentence[j]
		if c in p:
			p = p[c]
			if '' in p:
				yield sentence[i:j+1]
			j+=1
			if j>=N:
				i+=1
				j=i
				p=trie
		else:
			p = trie
			i+=1
			j=i


def calc(sentence,DAG,idx,route):
	N = len(sentence)
	route[N] = (1.0,'')
	for idx in xrange(N-1,-1,-1):
		candidates = [ ( FREQ.get(sentence[idx:x+1],min_freq) * route[x+1][0],x ) for x in DAG[idx] ]
		route[idx] = max(candidates)

def __cut_DAG(sentence):
	N = len(sentence)
	i,j=0,0
	p = trie
	DAG = {}
	while i<N:
		c = sentence[j]
		if c in p:
			p = p[c]
			if '' in p:
				if not i in DAG:
					DAG[i]=[]
				DAG[i].append(j)
			j+=1
			if j>=N:
				i+=1
				j=i
				p=trie
		else:
			p = trie
			i+=1
			j=i
	for i in xrange(len(sentence)):
		if not i in DAG:
			DAG[i] =[i]
	#pprint.pprint(DAG)
	route ={}
	calc(sentence,DAG,0,route=route)
	x = 0
	buf =u''
	while x<N:
		y = route[x][1]+1
		l_word = sentence[x:y]
		if y-x==1:
			buf+= l_word
		else:
			if len(buf)>0:
				if len(buf)==1:
					yield buf
					buf=u''
				else:
					regognized = finalseg.__cut(buf)
					for t in regognized:
						yield t
					buf=u''
			yield l_word		
		x =y

	if len(buf)>0:
		if len(buf)==1:
			yield buf
		else:
			regognized = finalseg.__cut(buf)
			for t in regognized:
				yield t


def cut(sentence,cut_all=False):
	if not ( type(sentence) is unicode):
		try:
			sentence = sentence.decode('utf-8')
		except:
			sentence = sentence.decode('gbk','ignore')
	re_han, re_skip = re.compile(ur"([\u4E00-\u9FA5]+)"), re.compile(ur"[^a-zA-Z0-9+#\n]")
	blocks = re_han.split(sentence)
	cut_block = __cut_DAG
	if cut_all:
		cut_block = __cut_all
	for blk in blocks:
		if re_han.match(blk):
				#pprint.pprint(__cut_DAG(blk))
				for word in cut_block(blk):
					yield word
		else:
			tmp = re_skip.split(blk)
			for x in tmp:
				if x!="":
					yield x

@@ -70,16 +70,47 @@
 ort time
+%0Aimport tempfile%0Aimport marshal
 %0A%0AFREQ =
@@ -152,31 +152,43 @@
 ):%0A%09
-global total%0A%09trie = %7B%7D
+lfreq = %7B%7D%0A%09trie = %7B%7D%0A%09ltotal = 0.0
 %0A%09co
@@ -323,20 +323,21 @@
 freq)%0A%09%09
-FREQ
+lfreq
 %5Bword%5D =
@@ -344,16 +344,17 @@
  freq%0A%09%09
+l
 total+=f
@@ -464,16 +464,30 @@
 urn trie
+, lfreq,ltotal
 %0A%0A%0A_curp
@@ -624,20 +624,455 @@
 .time()%0A
-trie
+cache_file = os.path.join(tempfile.gettempdir(),%22jieba.cache%22)%0Aload_from_cache_fail = True%0Aif os.path.exists(cache_file) and os.path.getmtime(cache_file)%3Eos.path.getmtime(os.path.join(_curpath,%22dict.txt%22)):%0A%09print %3E%3E sys.stderr, %22loading model from cache%22%0A%09try:%0A%09%09trie,FREQ,total,min_freq = marshal.load(open(cache_file,'rb'))%0A%09%09load_from_cache_fail = False%0A%09except:%0A%09%09load_from_cache_fail = True%0A%0Aif load_from_cache_fail:%0A%09trie,FREQ,total
  = gen_t
@@ -1110,16 +1110,17 @@
 .txt%22))%0A
+%09
 FREQ = d
@@ -1184,16 +1184,17 @@
 rmalize%0A
+%09
 min_freq
@@ -1219,16 +1219,81 @@
 lues())%0A
+%09marshal.dump((trie,FREQ,total,min_freq),open(cache_file,'wb'))%0A%0A
 print %3E%3E

use file cache to improve the loading speed after the first time of importing