import logging
import operator

from mapreduce.driver import MapReduceDriver
from mapreduce.emblem_freq import EmblemFreq
from nlp.emblem import Emblem
from processor.data_source import MongoDataSource

data_source = MongoDataSource()
songci_list = data_source.find()

logger = logging.getLogger('emblem_stat')
logging.basicConfig(level=logging.INFO)


def stat_freq():
    freq_threshold = 1

    driver_freq = MapReduceDriver(EmblemFreq.map_fn, EmblemFreq.reduce_fn)
    emblem_list = Emblem(songci_list).emblem_list()
    emblem_freq_stat = driver_freq(emblem_list)
    emblem_freq_stat.sort(key=operator.itemgetter(1), reverse=True)

    total_len = len(emblem_freq_stat)
    logger.info('Total length of emblem_freq_stat is %d', total_len)
    prev_freq = prev_freq_rate = 0  # cache previous quotient (a.k.a. freq) to improve performance
    for emblem_freq in emblem_freq_stat:  # todo: optimize this using concurrency
        emblem_name, freq = emblem_freq
        if freq <= freq_threshold:
            break
        freq_rate = prev_freq_rate if freq == prev_freq else freq / total_len
        data_source.save('emblem', {'name': emblem_name}, {'freq_rate': freq_rate})
        prev_freq = freq
        prev_freq_rate = freq_rate

    data_source.create_index('emblem', 'name', unique=True)
    data_source.create_index('emblem', 'freq_rate')


if __name__ == '__main__':
    stat_freq()

@@ -15,24 +15,76 @@
 %0Aimport 
-operator
+multiprocessing%0Aimport operator%0Afrom itertools import repeat
 %0A%0Afrom m
@@ -419,32 +419,8 @@
 ():%0A
-    freq_threshold = 1%0A%0A
     
@@ -765,16 +765,470 @@
 tal_len)
+%0A%0A    workers = (multiprocessing.cpu_count() or 1) * 4%0A    pool = multiprocessing.Pool(processes=workers)%0A    emblem_freq_chunks = MapReduceDriver.chunks(emblem_freq_stat, int(total_len / workers))%0A    pool.starmap(save_freq_stat, zip(emblem_freq_chunks, repeat(total_len)))%0A%0A    data_source.create_index('emblem', 'name', unique=True)%0A    data_source.create_index('emblem', 'freq_rate')%0A%0A%0Adef save_freq_stat(freq_stat, total_len):%0A    freq_threshold = 1
 %0A    pre
@@ -1342,23 +1342,16 @@
 freq in 
-emblem_
 freq_sta
@@ -1356,49 +1356,8 @@
 tat:
-  # todo: optimize this using concurrency
 %0A   
@@ -1629,24 +1629,24 @@
 freq = freq%0A
+
         prev
@@ -1673,121 +1673,8 @@
 te%0A%0A
-    data_source.create_index('emblem', 'name', unique=True)%0A    data_source.create_index('emblem', 'freq_rate')%0A%0A
 %0Aif 

optimize freq_stat.