<jupyter_start><jupyter_text>plot training curves in small grid env<jupyter_code>base_dir = '/Users/tchu/Documents/rl_test/signal_control_results'
plot_dir = base_dir + '/plots'
if not os.path.exists(plot_dir):
    os.mkdir(plot_dir)
color_cycle = sns.color_palette()
COLORS = {'ma2c': color_cycle[0], 'ia2c': color_cycle[1], 'iqll': color_cycle[2], 
          'iqld': color_cycle[3], 'greedy':color_cycle[4]}
TRAIN_STEP = 1e6
window = 40
def plot_train_curve(scenario='small_grid', date='jun06'):
    cur_dir = base_dir + ('/eval_%s/%s/train_data' % (date, scenario))
    names = ['global', 'local', 'neighbor']
    labels = ['Centralized A2C', 'Independent A2C', 'Multi-agent A2C']
    if scenario == 'large_grid':
        names = names[1:]
        labels = labels[1:]
    dfs = {}
    for file in os.listdir(cur_dir):
        name = file.split('_')[0]
        print(file + ', ' + name)
        if name in names:
            df = pd.read_csv(cur_dir + '/' + file)
            dfs[name] = df

    plt.figure(figsize=(10,8))
#     ymin = min([df.value.min() for df in dfs.values()])
#     ymax = max([df.value.max() for df in dfs.values()])
    ymin = []
    ymax = []
    xmin = min([df.step.min() for df in dfs.values()])
    for i, name in enumerate(names):
        df = dfs[name]
#         plt.plot(df.Step.values, df.Value.values, color=COLORS[i], linewidth=3, label=labels[i])
        x_mean = df.value.rolling(window).mean().values
        x_std = df.value.rolling(window).std().values
#         x_hi = df.value.rolling(window).max().values
        plt.plot(df.step.values, x_mean, color=COLORS[name], linewidth=3, label=labels[i])
        ymin.append(np.nanmin(x_mean - x_std))
        ymax.append(np.nanmax(x_mean + x_std))
        plt.fill_between(df.step.values, x_mean - x_std, x_mean + x_std, facecolor=COLORS[name], edgecolor='none', alpha=0.3)
    ymin = min(ymin)
    ymax = max(ymax)
    plt.xlim([xmin,TRAIN_STEP])
    plt.ylim([ymin * 1.05, ymax * 0.95])
    plt.xticks(fontsize=15)
    plt.yticks(fontsize=15)
    # plt.gca().xaxis.set_major_formatter(mtick.FormatStrFormatter('%.2e'))
    plt.xlabel('Training step', fontsize=20)
    plt.ylabel('Averaged episode reward', fontsize=20)
    plt.legend(loc='lower right', fontsize=20)
    plt.tight_layout()
    # plt.savefig(plot_dir + '/small_grid_train.png')
    plt.savefig(plot_dir + ('/%s_train.pdf' % scenario))
    plt.close()

    # calculate performance gains
    print('final performance wrt centralized agent:')
    ys = {}
    for name in names:
        y = dfs[name].value.values
        final = np.mean(y[-window:])
        init = np.mean(y[:window])
        gain = final - init
        print('%s: final: %.2f' % (name, final))
plot_train_curve()<jupyter_output>neighbor_train_reward.csv, neighbor
local_train_reward.csv, local
global_train_reward.csv, global
final performance wrt centralized agent:
global: final: -10.21
local: final: -9.34
neighbor: final: -9.16<jupyter_text>plot training curves in large grid env<jupyter_code>plot_train_curve(scenario='large_grid')<jupyter_output>neighbor_train_reward.csv, neighbor
local_train_reward.csv, local
final performance wrt centralized agent:
local: final: -197.56
neighbor: final: -142.29<jupyter_text>plot evaluation curves in small grid env<jupyter_code>episode_sec = 3600
def fixed_agg(xs, window, agg):
    xs = np.reshape(xs, (-1, window))
    if agg == 'sum':
        return np.sum(xs, axis=1)
    elif agg == 'mean':
        return np.mean(xs, axis=1)
    elif agg == 'median':
        return np.median(xs, axis=1)

def varied_agg(xs, ts, window, agg):
    t_bin = window
    x_bins = []
    cur_x = []
    for x, t in zip(list(xs) + [0], list(ts) + [episode_sec + 1]):
        if t <= t_bin:
            cur_x.append(x)
        else:
            if not len(cur_x):
                x_bins.append(0)
            else:
                if agg == 'sum':
                    x_stat = np.sum(np.array(cur_x))
                elif agg == 'mean':
                    x_stat = np.mean(np.array(cur_x))
                elif agg == 'median':
                    x_stat = np.median(np.array(cur_x))
                x_bins.append(x_stat)
            t_bin += window
            cur_x = [x]
    return np.array(x_bins)
    
def plot_series(df, name, tab, label, color, window=None, agg='sum', reward=False):
    episodes = list(df.episode.unique())
    num_episode = len(episodes)
    num_time = episode_sec
    print(label, name)
    print(df[name].describe())
    if reward:
        num_time = 720
    if window and (agg != 'mv'):
        num_time = num_time // window
    x = np.zeros((num_episode, num_time))
    for i, episode in enumerate(episodes):
        t_col = 'arrival_sec' if  tab == 'trip' else 'time_sec' 
        cur_df = df[df.episode == episode].sort_values(t_col)
        if window and (agg == 'mv'):
            cur_x = cur_df[name].rolling(window, min_periods=1).mean().values
        else:
            cur_x = cur_df[name].values    
        if window and (agg != 'mv'):
            if tab == 'trip':
                cur_x = varied_agg(cur_x, df[df.episode == episode].arrival_sec.values, window, agg)
            else:    
                cur_x = fixed_agg(cur_x, window, agg)
        print(cur_x.shape)
        x[i] = cur_x
    if num_episode > 1:
        x_mean = np.mean(x, axis=0)
        x_std = np.std(x, axis=0)
    else:
        x_mean = x[0]
        x_std = np.zeros(num_time)
    if (not window) or (agg == 'mv'):
        t = np.arange(1, episode_sec + 1)
        if reward:
            t = np.arange(10, episode_sec + 1, 5)
    else:
        t = np.arange(window, episode_sec + 1, window)
#     if reward:
#         print('%s: %.2f' % (label, np.mean(x_mean)))
    plt.plot(t, x_mean, color=color, linewidth=3, label=label)
    if num_episode > 1:
        x_lo = x_mean - x_std
        if not reward:
            x_lo = np.maximum(x_lo, 0)
        x_hi = x_mean + x_std
        plt.fill_between(t, x_lo, x_hi, facecolor=color, edgecolor='none', alpha=0.3)
        return np.nanmin(x_lo), np.nanmax(x_hi)
    else:
        return np.nanmin(x_mean), np.nanmax(x_mean)
    
def plot_combined_series(dfs, agent_names, col_name, tab_name, agent_labels, y_label, fig_name,
                         window=None, agg='sum', reward=False):
    plt.figure(figsize=(10,8))
    ymin = np.inf
    ymax = -np.inf
    for i, aname in enumerate(agent_names):
        df = dfs[aname][tab_name]
        y0, y1 = plot_series(df, col_name, tab_name, agent_labels[i], COLORS[aname], window=window, agg=agg,
                             reward=reward)
        ymin = min(ymin, y0)
        ymax = max(ymax, y1)
    
    plt.xlim([0, episode_sec])
    if (col_name == 'average_speed') and ('global' in agent_names):
        plt.ylim([0, 6])
    elif (col_name == 'wait_sec') and ('global' not in agent_names):
        plt.ylim([0, 3500])
    else:
        plt.ylim([ymin, ymax])
    plt.xticks(fontsize=15)
    plt.yticks(fontsize=15)
    plt.xlabel('Simulation time (sec)', fontsize=20)
    plt.ylabel(y_label, fontsize=20)
    if (col_name == 'wait_sec'):
        plt.legend(loc='upper left', fontsize=20)
    else:
        plt.legend(loc='best', fontsize=20)
    plt.tight_layout()
    plt.savefig(plot_dir + ('/%s.pdf' % fig_name))
    plt.close()
    
def sum_reward(x):
    x = [float(i) for i in x.split(',')]
    return np.sum(x)

def plot_eval_curve(scenario='large_grid', date='oct03'):
    cur_dir = base_dir + ('/eval_%s/%s/eva_data' % (date, scenario))
    names = ['greedy']
    labels = ['Greedy policy']
    dfs = {}
    for file in os.listdir(cur_dir):
        if not file.endswith('.csv'):
            continue
        if not file.startswith(scenario):
            continue
        name = file.split('_')[2]
        measure = file.split('_')[3].split('.')[0]
        if name in names:
            df = pd.read_csv(cur_dir + '/' + file)
#             if measure == 'traffic':
#                 df['ratio_stopped_car'] = df.number_stopped_car / df.number_total_car * 100
#             if measure == 'control':
#                 df['global_reward'] = df.reward.apply(sum_reward)
            if name not in dfs:
                dfs[name] = {}
            dfs[name][measure] = df
    
    # plot avg queue
    plot_combined_series(dfs, names, 'avg_queue', 'traffic', labels,
                         'Average queue length (veh)', scenario + '_queue', window=60, agg='mv')
    # plot avg speed
    plot_combined_series(dfs, names, 'avg_speed_mps', 'traffic', labels,
                         'Average car speed (m/s)', scenario + '_speed', window=60, agg='mv')
    # plot avg waiting time
    plot_combined_series(dfs, names, 'avg_wait_sec', 'traffic', labels,
                         'Average waiting time (sec)', scenario + '_wait', window=60, agg='mv')
    # plot trip completion
    plot_combined_series(dfs, names, 'number_arrived_car', 'traffic', labels,
                         'Trip completion rate (veh/5 min)', scenario + '_tripcomp', window=300, agg='sum')
    # plot trip time
    plot_combined_series(dfs, names, 'duration_sec', 'trip', labels,
                         'Avg trip time (sec)', scenario + '_triptime', window=60, agg='mean')
    # plot trip waiting time
    plot_combined_series(dfs, names, 'wait_sec', 'trip', labels,
                         'Avg trip delay (sec)', scenario + '_tripwait', window=60, agg='mean')
    plot_combined_series(dfs, names, 'reward', 'control', labels,
                         'Step reward', scenario + '_reward', reward=True, window=6, agg='mv')
plot_eval_curve()<jupyter_output>Greedy policy avg_queue
count    3600.000000
mean        3.807843
std         2.042572
min         0.080000
25%         1.791667
50%         3.273333
75%         5.820000
max         7.966667
Name: avg_queue, dtype: float64
(3600,)
Greedy policy avg_speed_mps
count    3600.000000
mean        2.121568
std         1.165127
min         0.000000
25%         1.087307
50%         1.897988
75%         2.845419
max         5.743651
Name: avg_speed_mps, dtype: float64
(3600,)
Greedy policy avg_wait_sec
count    3600.000000
mean       35.639628
std        23.644824
min         0.000000
25%        14.987172
50%        30.097387
75%        57.866262
max        90.487483
Name: avg_wait_sec, dtype: float64
(3600,)
Greedy policy number_arrived_car
count    3600.000000
mean        1.115278
std         1.112604
min         0.000000
25%         0.000000
50%         1.000000
75%         2.000000
max         7.000000
Name: number_arrived_car, dtype: float64
(12,)
Greedy policy duration_sec
count    4016.0[...]<jupyter_text>plot evaluation curves in large grid env<jupyter_code>plot_eval_curve(scenario='large_grid')<jupyter_output>Independent A2C: -268.44
Multi-agent A2C: -124.13
Greedy policy: -207.65
StarcoderdataJupyterStructuredCleanDedup