<jupyter_start><jupyter_text>Human and SOTA performance on Charade-STA<jupyter_code>def performance_comparison_charade():
    test_data = CharadesSTA2Instances(pd.read_csv('data/processed/test.csv'))
    result_2dtan = pkl.load(open("reports/figure_src/results/charade/2d-tan.pkl", "rb"))
    result_scdm = pkl.load(open("reports/figure_src/results/charade/scdm.pkl", "rb"))
    result_human = pkl.load(open("reports/outputs/performance/charade-sta/representative_human.pkl", "rb"))
    result_rand_human = json.load(open('reports/outputs/performance/charade-sta/random_sample_human.json'))
    
    subresult_2dtan = []
    for item in result_2dtan:
        for key, _, _, _ in result_human:
            if key == item[0]:
                subresult_2dtan.append(item)
                break

    subresult_scdm = []
    for item in result_scdm:
        for key, _, _, _ in result_human:
            if key == item[0]:
                subresult_scdm.append(item)
                break

    leaderboard = {
        "SCDM": (accumulate_metrics(subresult_scdm)['R@1 IoU>0.5']*100, 0),
        "2D-TAN": (accumulate_metrics(subresult_2dtan)['R@1 IoU>0.5']*100, 0),
        "Human": (np.mean(result_rand_human['R@1 IoU>0.5']), np.std(result_rand_human['R@1 IoU>0.5'])),
        "Rep. Human": (accumulate_metrics(result_human)['R@1 IoU>0.5']*100, 0),
    }
    return leaderboard<jupyter_output><empty_output><jupyter_text>Human and SOTA performance on ActivityNet<jupyter_code>def performance_comparison_activitynet():
    raw_data = json.load(open("data/raw/ActivityNet/val_2.json"))
    test_data = ActivityNetCap2Instances(raw_data)
    result_2dtan = pkl.load(open("reports/figure_src/results/activitynet/2d-tan.pkl", "rb"))
    result_scdm = pkl.load(open("reports/figure_src/results/activitynet/scdm.pkl", "rb"))
    result_human = pkl.load(open("reports/outputs/performance/activitynet/representative_human.pkl", "rb"))
    result_rand_human = json.load(open('reports/outputs/performance/activitynet/random_sample_human.json'))
    
    subresult_2dtan = []
    for item in result_2dtan:
        for key, _, _, _ in result_human:
            if key == item[0]:
                subresult_2dtan.append(item)
                break

    subresult_scdm = []
    for item in result_scdm:
        for key, _, _, _ in result_human:
            if key == item[0]:
                subresult_scdm.append(item)
                break

    leaderboard = {
        "SCDM": (accumulate_metrics(subresult_scdm)['R@1 IoU>0.5']*100, 0),
        "2D-TAN": (accumulate_metrics(subresult_2dtan)['R@1 IoU>0.5']*100, 0),
        "Human": (np.mean(result_rand_human['R@1 IoU>0.5']), np.std(result_rand_human['R@1 IoU>0.5'])),
        "Rep. Human": (accumulate_metrics(result_human)['R@1 IoU>0.5']*100, 0),
    }
    return leaderboard
charade_leaderboard = performance_comparison_charade()
activitynet_leaderboard = performance_comparison_activitynet()
def display_score(bar, ax, top_margin=4,color="w"):
    ax.text(
        bar.get_x()+bar.get_width()*.5,
        bar.get_height()-top_margin,
        f"{bar.get_height():.1f}",
        horizontalalignment="center",
        fontsize=14,
        color=color
    )

sns.set_style("white")
f, axes = plt.subplots(1, 2, figsize=(14, 3), sharey=True)
top_margin = 5
for ax, leaderboard in zip(axes, [charade_leaderboard, activitynet_leaderboard]):
    labels = list(leaderboard.keys())
    scores = [v[0] for v in leaderboard.values()]
    std = [v[1] for v in leaderboard.values()]
    n = len(labels)
    print(std)

    bars = ax.bar(np.arange(n), scores, color="#606060", width=0.5, yerr=std)
    ax.set_xticks(np.arange(n))
    ax.set_xticklabels(labels, fontsize=14)

    bars[2].set_color("#6002ee")
    bars[3].set_color("#6002ee")
    
    for i, b in enumerate(bars):
        display_score(b, ax, top_margin, color="w")

    ax.yaxis.grid()
    
sns.despine(left=True)
axes[0].set_title("Charade-STA", fontsize=15)
axes[1].set_title("ActivityNet Captions", fontsize=15)
plt.savefig('reports/figures/performance_comparison/performance_SOTA_and_Human.pdf',
           bbox_inches='tight')<jupyter_output>[0, 0, 1.0567549384791168, 0]
[0, 0, 1.1728894338309637, 0]
StarcoderdataJupyterStructuredCleanDedup