# encoding=utf-8
import jieba

filePath='/home/tutumeimei1023/DeepQA2/data/text/content.txt'
fileSegWordDonePath ='/home/tutumeimei1023/DeepQA2/data/text/contentSegDone.txt'
# read the file by line
fileTrainRead = []
#fileTestRead = []
with open(filePath) as fileTrainRaw:
    for line in fileTrainRaw:
        fileTrainRead.append(line)


# define this function to print a list with Chinese
def PrintListChinese(list):
    for i in range(len(list)):
        print (list[i]),
# segment word with jieba
fileTrainSeg=[]
for i in range(len(fileTrainRead)):
    line = jieba.cut(fileTrainRead[i], cut_all=False)
    fileTrainSeg.append(line)
    if i % 100 == 0 :
        print (i)

# to test the segment result
#PrintListChinese(fileTrainSeg[10])

# save the result
with open(fileSegWordDonePath,'wb') as fW:
    for i in range(len(fileTrainSeg)):
        fW.write(fileTrainSeg[i][0].encode('utf-8'))
        fW.write('\n')
@@ -1,920 +1,527 @@
-# encoding=utf-8%0Aimport jieba%0A%0AfilePath='/home/tutumeimei1023/DeepQA2/data/text/content.txt'%0AfileSegWordDonePath ='/home/tutumeimei1023/DeepQA2/data/text/contentSegDone.txt'%0A# read the file by line%0AfileTrainRead = %5B%5D%0A#fileTestRead = %5B%5D%0Awith open(filePath) as fileTrainRaw:%0A    for line in fileTrainRaw:%0A        fileTrainRead.append(line)%0A%0A%0A# define this function to print a list with Chinese%0Adef PrintListChinese(list):%0A    for i in range(len(list)):%0A        print (list%5Bi%5D),%0A# segment word with jieba%0AfileTrainSeg=%5B%5D%0Afor i in range(len(fileTrainRead)):%0A    line = jieba.cut(fileTrainRead%5Bi%5D, cut_all=False)
+import sys%0Asys.path.append('../')%0A%0Aimport jieba%0Aimport jieba.analyse%0Afrom optparse import OptionParser%0A%0AUSAGE = %22usage:    python extract_tags.py %5Bfile name%5D -k %5Btop k%5D%22%0A%0Aparser = OptionParser(USAGE)%0Aparser.add_option(%22-k%22, dest=%22topK%22)%0Aopt, args = parser.parse_args()%0A%0A%0Aif len(args) %3C 1:%0A    print(USAGE)%0A    sys.exit(1)%0A%0Afile_name = args%5B0%5D%0A%0Aif opt.topK is None:%0A    topK = 10%0Aelse:
 %0A    
-fileTrainSeg.append(line)%0A    if i %25 100 == 0 :%0A        print (i)%0A%0A# to test the segment result%0A#PrintListChinese(fileTrainSeg%5B10%5D)%0A%0A# save the result%0Awith open(fileSegWordDonePath,'wb') as fW:%0A    for i in range(len(fileTrainSeg)):%0A        fW.write(fileTrainSeg%5Bi%5D%5B0%5D.encode('utf-8'))%0A        fW.write('%5Cn'
+topK = int(opt.topK)%0A%0Acontent = open(file_name, 'rb').read()%0A%0Atags = jieba.analyse.extract_tags(content, topK=topK)%0A%0Aprint(%22,%22.join(tags)
 )

test lightweight