@@ -0,0 +1,1870 @@
+#!/usr/bin/env python%0A# -*- coding: utf-8 -*-%0A%0A# Copyright (c) 2014 J%C3%A9r%C3%A9mie DECOCK (http://www.jdhp.org)%0A%0A# run:%0A#   mpirun -np 4 python scatter_gather_pickle_big_data.py %7C sort%0A#     or%0A#   mpiexec -n 4 python scatter_gather_pickle_big_data.py %7C sort%0A%0Afrom mpi4py import MPI%0Aimport itertools%0Aimport math%0A%0A###############################################################################%0A%0Adef grouper(iterable, num_items, fillvalue=None): # TODO%0A    '''Collect data into fixed-length chunks or blocks%0A%0A    e.g. grouper('ABCDEFG', 3, 'x') --%3E ABC DEF Gxx%0A    See: http://stackoverflow.com/questions/10124751/convert-a-flat-list-to-list-of-list-in-python'''%0A    args = %5Biter(iterable)%5D * num_items%0A    return list(itertools.izip_longest(fillvalue=fillvalue, *args))%0A%0A%0Adef flatten(l):%0A    '''Flatten a list (eg. %5B%5B1,2,3%5D,%5B4,5,6%5D, %5B7%5D, %5B8,9%5D%5D -%3E %5B1,2,3,4,5,6,7,8,9%5D).'''%0A    if l is None:%0A        return None%0A    else:%0A        return %5Bitem for sublist in l for item in sublist%5D%0A%0A###############################################################################%0A%0A# INPUT%0A%0Ainput_data = %5B'x' + str(i) for i in range(15)%5D%0Aprint %22%5B1. input%5D%22, input_data%0A%0A# MPI%0A%0Acomm = MPI.COMM_WORLD%0Asize = comm.Get_size()%0Arank = comm.Get_rank()%0A%0Aif rank == 0:%0A    data_len = 16%0A    data = input_data%0A    num_item_per_proc = int(math.ceil(float(len(data)) / size)) # TODO%0A    data = grouper(data, num_item_per_proc)%0Aelse:%0A    data = None%0A%0A# SCATTER%0A%0Aprint %22%5B2. before scatter%5D process%22, rank, %22:%22, data%0A%0Adata = comm.scatter(data, root=0)%0A%0Aprint %22%5B3. after scatter%5D process%22, rank, %22:%22, data%0A%0A# PROCESS DATA%0A%0Adata = %5Bitem.replace('x', 'y') for item in data if item is not None%5D%0A%0A# GATHER%0A%0Aprint %22%5B4. before gather%5D process%22, rank, %22:%22, data%0A%0Adata = comm.gather(data, root=0)%0A%0Aprint %22%5B5. after gather%5D process%22, rank, %22:%22, data%0A%0A# OUTPUT%0A%0Aoutput_data = flatten(data)%0Aprint %22%5B6. output%5D%22, output_data%0A%0A

Add scatter_gather_pickle_big_data.py.