@@ -0,0 +1,1554 @@
+def map(func, local_args, global_arg=None,%0A        chunksize=1, processes=2):%0A    '''%0A    global_arg: a global variable shared with all processes intended to be read-only%0A    so that we can utilize the copy-on-write mechanism in linux to avoid unnecceary copy of data,%0A    this is good for parallel processing of large data structure, like a big pandas dataframe (pass%0A    df as global_arg to avoid unncessary copy, and indices of subset as local_args that is to be copied)%0A    '''%0A    global_arg_name = None%0A    try:%0A        if global_arg is not None:%0A            global_arg_name = random_string(10, prefix='_tmp_global_arg')%0A            globals()%5Bglobal_arg_name%5D = global_arg%0A            def func_with_global(local_args, global_arg_name):%0A                global_arg = globals()%5Bglobal_arg_name%5D %0A                return func(local_args, global_arg)%0A                %0A            process_func = toolz.partial(%0A                func_with_global,%0A                global_arg_name=global_arg_name)%0A        else:%0A            process_func = func%0A%0A        if processes == 1:%0A            result = list(toolz.map(process_func, local_args))%0A        else:%0A            import pathos.multiprocessing as mp%0A            pool = mp.Pool(processes=processes)%0A            result = pool.map(process_func, local_args, chunksize=chunksize)%0A            pool.close()%0A            pool.join()%0A%0A        return result%0A    except Exception as ex:%0A        print 'exception: %7B%7D'.format(ex)%0A    finally:%0A        if global_arg_name is not None:%0A            del globals()%5Bglobal_arg_name%5D%0A

create parmap.py