<jupyter_start><jupyter_text>Using Joblib<jupyter_code>joblib_procs = []
joblib_compute_mean = []
joblib_compute_std = []
joblib_write_mean = []
joblib_write_std = []
joblib_total_times = []

prefix = 'GIVJoblib'
for file_name in os.listdir('.'):
    if not file_name.startswith(prefix): 
        continue
    nodes, ranks_per_node = parse_log_filename(file_name, prefix)
    compute_times, write_times, tot_time = parse_console_logs(file_name)
    """
    Note that compute time is measured for 1 rank
    It is going to look like just on worker finished so much work in X time
    In reality N workers together brought down the time. so:
    time_for_one_worker = time_for_n_workers * num_workers
    
    In mpi4py, each worker is independently profiled / timed whereas this is NOT possible in joblib
    What we can do for an apples-to-apples comparison is to scale the measured time per node by the number of workers
    """
    compute_times = np.array(compute_times) * ranks_per_node
    # Write times are indeed just ONE process writing to the file so no conversion is necessary
    joblib_procs.append(ranks_per_node)
    joblib_compute_mean.append(np.mean(compute_times))
    joblib_compute_std.append(np.std(compute_times))
    joblib_write_mean.append(np.mean(write_times))
    joblib_write_std.append(np.std(write_times))
    joblib_total_times.append(tot_time)

# This can be of any order so sort based on ranks:
sort_order = np.array(joblib_procs).argsort()

joblib_procs = np.array(joblib_procs)[sort_order]
joblib_compute_mean = np.array(joblib_compute_mean)[sort_order]
joblib_compute_std = np.array(joblib_compute_std)[sort_order]
joblib_write_mean = np.array(joblib_write_mean)[sort_order]
joblib_write_std = np.array(joblib_write_std)[sort_order]
joblib_total_times = np.array(joblib_total_times)[sort_order]
fig, axes = plt.subplots(ncols=3, figsize=(12, 4))
axes.flat[0].set_ylabel('Time (sec)', fontsize=16)
for axis, mean_vals, std_vals, qty_name in zip(axes.flat, 
                                              [joblib_compute_mean, joblib_write_mean, joblib_total_times],
                                              [joblib_compute_std, joblib_write_std, [0 for item in joblib_total_times]],
                                              ['Compute time/pixel/worker', 'Write time/pixel', 'Total time']):
    #axis.scatter(ranks, mean_vals)
    axis.errorbar(joblib_procs, mean_vals, yerr=std_vals, fmt='o')
    axis.set_xscale('log')
    axis.axvline(x=32, color='grey', ls=':')
    axis.set_title(qty_name, fontsize=16)
    axis.set_xlabel('Python Processes', fontsize=16)
    
usid.plot_utils.set_tick_font_size(axes, 14)
fig.tight_layout()<jupyter_output><empty_output><jupyter_text>Comparing joblib and mpi4py Write times* In the case of joblib, the entire data is written by one process regardless of number of workers. So, the write time in theory should be invariant and this is what we see.* It is cheapest to write all the data once. Writing N small chunks will be unnessarily slow. Moreover, N workers competing to write a chunk will make writes even slower. Therefore, in the case of mpi4py, the write time is expected to scale with the number of ranks. This is what we see. Compute times* In the case of joblib, compute time is measured for 1 (the only) rank. It is going to look like just on worker finished so much work in X time. In reality N workers together brought down the time. In mpi4py, each worker is independently profiled / timed whereas this is NOT possible in joblib. What we can do for an apples-to-apples comparison is to scale the measured time per node by the number of workers* Why exactly do we see compute time per worker per pixel rise with the number of workers?        * One unfair comparison is that each run gave each worker different numbers of jobs. In fact, the product of the number of jobs per worker and the number of workers is constant.    * Is it the cost associated with adding more workers? One way to find out is to simply fix the number of jobs for all workers.  Total time* The most expensive operation is the compute time. Write time is negligible for small scales. Compute times for both mpi4py and joblib are comparable* Total time is therefore identical whether the compute was through joblib or mpi4py<jupyter_code>fig, axes = plt.subplots(ncols=3, figsize=(12, 4))
axes.flat[0].set_ylabel('Time (sec)', fontsize=16)
axes.flat[1].set_yscale('log')
for axis, mean_vals in zip(axes.flat, [mpi_compute_mean, mpi_write_mean, mpi_total_times]):
    axis.scatter(mpi_ranks[:5], mean_vals[:5], color='orange', s=75, label='mpi4py')
    
for axis, mean_vals, qty_name in zip(axes.flat, 
                                     [joblib_compute_mean, joblib_write_mean, joblib_total_times],
                                     ['Compute time/pixel/worker', 'Write time/pixel', 'Total time']):
    #axis.scatter(ranks, mean_vals)
    axis.scatter(joblib_procs, mean_vals, label='joblib')
    axis.set_xscale('log')
    axis.set_title(qty_name, fontsize=16)
    axis.set_xlabel('Workers', fontsize=16)
    axis.legend(fontsize=16)
    
usid.plot_utils.set_tick_font_size(axes.flat, 14)
fig.tight_layout()
num_jobs = 68096
tot_time_per_pixel = np.array(joblib_compute_mean)  * joblib_procs + np.array(joblib_write_mean)
estimated_serial_time = num_jobs * tot_time_per_pixel
joblib_estimated_parallel_time = estimated_serial_time / np.array(joblib_procs)

fig, axis = plt.subplots(figsize=(5.5, 5))
axis.scatter(joblib_procs, joblib_total_times, s=90, label='Measured')
axis.scatter(joblib_procs, joblib_estimated_parallel_time, label='comp+write')
axis.set_xscale('log')
axis.axvline(x=32, color='grey', ls=':')
axis.set_xlabel('Python Processes', fontsize=16)
axis.set_ylabel('Execution time (sec)', fontsize=16)
axis.legend(fontsize=16)
axis.set_title('Compute + Write make up all the time')
usid.plot_utils.set_tick_font_size(axis, 14)
fig.tight_layout()<jupyter_output><empty_output><jupyter_text>Constant jobs per workerJobs are scaled with the number of cores ready to do work. Each process gets the **same** number of jobs (1,000) to work on<jupyter_code>def parse_logs_for_series(prefix): 
    scaled_procs = []
    scaled_compute_mean = []
    scaled_compute_std = []
    scaled_write_mean = []
    scaled_write_std = []
    scaled_total_times = []

    for file_name in os.listdir('.'):
        if not file_name.startswith(prefix): 
            continue
        nodes, ranks_per_node = parse_log_filename(file_name, prefix)
        compute_times, write_times, tot_time = parse_console_logs(file_name)
        """
        Note that compute time is measured for 1 rank
        It is going to look like just on worker finished so much work in X time
        In reality N workers together brought down the time. so:
        time_for_one_worker = time_for_n_workers * num_workers

        In mpi4py, each worker is independently profiled / timed whereas this is NOT possible in joblib
        What we can do for an apples-to-apples comparison is to scale the measured time per node by the number of workers
        """
        compute_times = np.array(compute_times) * ranks_per_node
        # Write times are indeed just ONE process writing to the file so no conversion is necessary
        scaled_procs.append(ranks_per_node)
        scaled_compute_mean.append(np.mean(compute_times))
        scaled_compute_std.append(np.std(compute_times))
        scaled_write_mean.append(np.mean(write_times))
        scaled_write_std.append(np.std(write_times))
        scaled_total_times.append(tot_time)

    # This can be of any order so sort based on ranks:
    sort_order = np.array(scaled_procs).argsort()

    scaled_procs = np.array(scaled_procs)[sort_order]
    scaled_compute_mean = np.array(scaled_compute_mean)[sort_order]
    scaled_compute_std = np.array(scaled_compute_std)[sort_order]
    scaled_write_mean = np.array(scaled_write_mean)[sort_order]
    scaled_write_std = np.array(scaled_write_std)[sort_order]
    scaled_total_times = np.array(scaled_total_times)[sort_order]
    
    return scaled_procs, scaled_compute_mean, scaled_compute_std, scaled_write_mean, scaled_write_std, scaled_total_times
scaled_procs, scaled_compute_mean, _, scaled_write_mean, _, scaled_total_times = parse_logs_for_series('GIV1xscaledJobs')
fig, axes = plt.subplots(ncols=3, figsize=(12, 4))
axes.flat[0].set_ylabel('Time (sec)', fontsize=16)
axes.flat[1].set_yscale('log')
for axis, mean_vals, qty_name in zip(axes.flat, 
                                     [scaled_compute_mean, scaled_write_mean, scaled_total_times],
                                     ['Compute time/pixel/worker', 'Write time/pixel', 'Total time']):
    #axis.scatter(ranks, mean_vals)
    axis.scatter(scaled_procs, mean_vals)
    axis.set_xscale('log')
    axis.set_title(qty_name, fontsize=16)
    axis.set_xlabel('Python Processes', fontsize=16)
    
usid.plot_utils.set_tick_font_size(axes, 14)
fig.tight_layout()<jupyter_output><empty_output><jupyter_text>Doubling the number of jobs per process:<jupyter_code>scaled_2_procs, scaled_2_compute_mean, _, scaled_2_write_mean, _, scaled_2_total_times = parse_logs_for_series('GIV2xscaledJobs')
scaled_3_procs, scaled_3_compute_mean, _, scaled_3_write_mean, _, scaled_3_total_times = parse_logs_for_series('GIV3xscaledJobs')<jupyter_output><empty_output><jupyter_text>The computational time / pixel / worker is unaffected by the load and is the same for a given number of workers (MPI or otherwise).<jupyter_code>fig, axes = plt.subplots(ncols=3, figsize=(12, 4))
axes.flat[0].set_ylabel('Time (sec)', fontsize=16)
axes.flat[1].set_yscale('log')
for axis, jobs_1, jobs_2, jobs_3, qty_name in zip(axes.flat, 
                                                  [scaled_compute_mean, scaled_write_mean, scaled_total_times],
                                                  [scaled_2_compute_mean, scaled_2_write_mean, scaled_2_total_times],
                                                  [scaled_3_compute_mean, scaled_3_write_mean, scaled_3_total_times],
                                                  ['Compute time/pixel/worker', 'Write time/pixel', 'Total time']):
    #axis.scatter(ranks, mean_vals)
    axis.scatter(scaled_procs, jobs_1, color='black', label='1k')
    axis.scatter(scaled_2_procs, jobs_2, color='red', label='2k')
    axis.scatter(scaled_3_procs, jobs_3, color='green', label='3k')
    axis.set_xscale('log')
    axis.legend(fontsize=14)
    axis.set_title(qty_name, fontsize=16)
    axis.set_xlabel('Python Processes', fontsize=16)
    
usid.plot_utils.set_tick_font_size(axes, 14)
fig.tight_layout()<jupyter_output><empty_output><jupyter_text>Constant jobs<jupyter_code>const_procs, const_compute_mean, _, const_write_mean, _, const_total_times = parse_logs_for_series('GIVconstJobs')
fig, axes = plt.subplots(ncols=2, figsize=(8, 4))
axes.flat[0].set_ylabel('Time (sec)', fontsize=16)
for axis, mean_vals, qty_name in zip(axes.flat, 
                                              [const_compute_mean, const_total_times],
                                              ['Compute time/pixel/worker', 'Total time']):
    #axis.scatter(ranks, mean_vals)
    axis.scatter(const_procs, mean_vals)
    axis.set_xscale('log')
    axis.set_title(qty_name, fontsize=16)
    axis.set_xlabel('Python Processes', fontsize=16)
    
usid.plot_utils.set_tick_font_size(axes, 14)
fig.tight_layout()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup