<jupyter_start><jupyter_text>Prediction MetricsFor PeTaL Labeller Issue 42 (https://github.com/nasa-petal/PeTaL-labeller/issues/42), a start to analyzing precision, recall, &c., of MATCH.<jupyter_code>!pip3 install --upgrade scikit-learn
import os
from datetime import datetime

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from matplotlib import rc
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.metrics import multilabel_confusion_matrix

from collections import namedtuple
from tqdm import tqdm
# If running on Google Colab: Mount drive and cd to workspace.
from google.colab import drive
drive.mount('/content/drive')

# IMPORTANT: Replace DRIVE_PATH with the path you plan to clone MATCH into.
# If you are in the PeTaL Drive and your PeTAL link is under MyDrive/
# then the following DRIVE_PATH should work fine?
# DRIVE_PATH = '/content/drive/MyDrive/PeTaL/PeTaL Data/MATCH on PeTaL Data/'
DRIVE_PATH = '/content/drive/Shareddrives/NASA'
%cd $DRIVE_PATH
if not os.path.exists('MATCH/'):
    !git clone https://github.com/yuzhimanhua/MATCH.git
else:
    print("You have already cloned the MATCH repository.")
%cd ./MATCH
!ls
# Install requirements in requirements.txt
# !chmod 755 -R .
!pip3 install -r requirements.txt
DATASET = "PeTaL"
MODEL = "MATCH"<jupyter_output><empty_output><jupyter_text>Calculation of precision and recall<jupyter_code>res_labels = np.load(f"{DATASET}/results/{MODEL}-{DATASET}-labels.npy", allow_pickle=True)
res_scores = np.load(f"{DATASET}/results/{MODEL}-{DATASET}-scores.npy", allow_pickle=True)
test_labels = np.load(f"{DATASET}/test_labels.npy", allow_pickle=True)
train_labels = np.load(f"{DATASET}/train_labels.npy", allow_pickle=True)
Stats = namedtuple("Stats", "threshold topk precision recall f1")

"""
    compute_stats(threshold)

    Parameters:
        threshold: float, 0.0 < threshold < 1.0
        res_labels: numpy array of predicted labels
        res_scores: numpy array of predicted label scores
        test_labels: numpy array of target labels
    
    Returns:
        Stats object containing
            threshold
            topk: average number of labels above threshold
            precision: average precision across examples
            recall: average recall across examples
            f1: average F1 score across examples
    
    Note:
        precision, recall, and F1 scores are macro (averaged across examples, not labels)
"""
def compute_stats(threshold, res_labels, res_scores, test_labels):
    precisions = []
    recalls = []
    topks = []
    f1s = []
    for res_label, res_score, test_label in zip(res_labels, res_scores, test_labels):
        topk = np.argmax(res_score < threshold) # topk becomes the number of labels scoring above the threshold
        precision = 1.0 if topk == 0 else np.mean([1 if x in test_label else 0 for x in res_label[:topk]])
        recall = np.mean([1 if x in res_label[:topk] else 0 for x in test_label])
        f1 = 0 if (precision + recall) == 0 else (2 * precision * recall) / (precision + recall)
        topks.append(topk)
        precisions.append(precision)
        recalls.append(recall)
        f1s.append(f1)
        # print(res_label[:topk], precision, recall)
    return Stats(threshold, np.mean(topks), np.mean(precisions), np.mean(recalls), np.mean(f1s))
    # print("Threshold:", threshold)
    # print("Average number of labels:", np.mean(topks))
    # print("Average precision:", np.mean(precisions))
    # print("Average recall:", np.mean(recalls))
    # print("Average F1 score:", np.mean(f1s))
thresholds = list(x / 10000 for x in range(1, 10)) + \
    list(x / 1000 for x in range(1, 10)) + \
    list(x / 100 for x in range(1, 10)) + \
    list(x / 20 for x in range(2, 19)) + \
    list((90 + x) / 100 for x in range(1, 10)) + \
    list((990 + x) / 1000 for x in range(1, 10)) + \
    list((9990 + x) / 10000 for x in range(1, 10))

ps = []
rs = []
ts = []
f1s = []
topks = []
for threshold in tqdm(thresholds):
    stats = compute_stats(threshold, res_labels, res_scores, test_labels)
    ps.append(stats.precision)
    rs.append(stats.recall)
    ts.append(threshold)
    f1s.append(stats.f1)
    topks.append(stats.topk)<jupyter_output>100%|██████████| 71/71 [00:00<00:00, 93.61it/s]<jupyter_text>Make plotsIn this section we make the following plots to assess the performance of the model.- Precision-recall curve- Precision, recall, and F1 Score by varying threshold- Number of labels predicted by varying threshold<jupyter_code>GENERATE_PLOTS = False
ALL_PLOTS_PATH = '../plots/'
if not os.path.exists(ALL_PLOTS_PATH):
    os.mkdir(ALL_PLOTS_PATH)
else:
    print(f"You already have a plots directory at {ALL_PLOTS_PATH}.")
time_str = datetime.now().strftime("%Y%m%d_%H%M%S")
comment = "on_MAG_CS"
PLOTS_PATH = os.path.join(ALL_PLOTS_PATH, f"{time_str}_{comment}")

if not os.path.exists(PLOTS_PATH):
    os.mkdir(PLOTS_PATH)
    print(f"New plots directory at {PLOTS_PATH}")
else:
    print(f"You already have a plots directory at {PLOTS_PATH}")
plt.grid()
plt.title(f'Precision-Recall Curve for {MODEL} on {DATASET}, varying threshold')
plt.plot(ps, rs, linestyle='-')
plt.xlabel('Recall')
plt.xlim(0, 1)
plt.ylabel('Precision')
plt.ylim(0, 1)

if GENERATE_PLOTS:
    PLOT_PATH = os.path.join(PLOTS_PATH, f'prc_{MODEL}_{DATASET}.png')
    plt.savefig(fname=PLOT_PATH, facecolor='w', transparent=False)
    print(f"Your plot is saved as {PLOT_PATH}")
plt.grid()
plt.title(f'Precision, Recall, and F1 Score by Threshold for {MODEL} on {DATASET}')
plt.plot(ts, ps, linestyle='-', label='Precision')
plt.plot(ts, rs, linestyle='-', label='Recall')
plt.plot(ts, f1s, linestyle='-', label='F1 score')
plt.xlabel('Threshold')
plt.xlim(0, 1)
plt.ylabel('Metrics')
plt.ylim(0, 1)
plt.legend()

if GENERATE_PLOTS:
    PLOT_PATH = os.path.join(PLOTS_PATH, f'prf1_{MODEL}_{DATASET}.png')
    plt.savefig(fname=PLOT_PATH, facecolor='w', transparent=False)
    print(f"Your plot is saved as {PLOT_PATH}")
plt.grid()
plt.title(f'Number of Labels Predicted by Threshold for {MODEL} on {DATASET}')
plt.plot(ts, topks, linestyle='-', label='Number of Labels')
plt.xlabel('Threshold')
plt.xlim(0, 1)
plt.ylabel('Labels')
plt.legend()

if GENERATE_PLOTS:
    PLOT_PATH = os.path.join(PLOTS_PATH, f'labels_{MODEL}_{DATASET}.png')
    plt.savefig(fname=PLOT_PATH, facecolor='w', transparent=False)
    print(f"Your plot is saved as {PLOT_PATH}")<jupyter_output>Your plot is saved as ../plots/20210701_173537_on_MAG_CS/labels_MATCH_MAG.png<jupyter_text>Additional analysis<jupyter_code>parent_labels = set()
with open('PeTaL/taxonomy.txt', 'r') as f:
    for line in f:
        labels = line.split()
        parent_labels.add(labels[0])
parents = 0
total = 0
for labels in res_labels:
    for label in labels[:5]:
        total += 1
        if label in parent_labels:
            parents += 1
print(f"{parents} predicted labels out of {total}, or {parents / total :.3f}, are parent (non-leaf) labels.")

parents = 0
total = 0
for labels in test_labels:
    for label in labels:
        total += 1
        if label in parent_labels:
            parents += 1
print(f"{parents} target labels out of {total}, or {parents / total :.3f}, are parent (non-leaf) labels.")
ks = list(range(1, 26))
pct_parents = []
for k in ks:
    parents = 0
    total = 0
    for labels in res_labels:
        for label in labels[:k]:
            total += 1
            if label in parent_labels:
                parents += 1
    pct_parents.append(parents / total)
    print(f"{parents} predicted labels out of {total}, or {parents / total :.3f}, are parent (non-leaf) labels.")   
plt.title("Proportion of parent labels in top k predicted labels")
plt.plot(ks, pct_parents, color='b', label="Predicted labels")
plt.xlim(1, 25)
plt.xlabel("k")
plt.ylim(0, 1)
plt.ylabel("Proportion of parent labels")
plt.grid()
plt.axhline(y=0.561, color='r')
plt.savefig(fname="are_parent_labels_overrepresented.png", facecolor='w', transparent=False)<jupyter_output><empty_output><jupyter_text>Generate confusion matrices<jupyter_code>k = 5
test_labels
# top_res_labels = res_labels[:,:k]
threshold = 0.1
# np.argmax(res_score < threshold) becomes the number of labels scoring above the threshold
top_res_labels = np.array(list(res_label[:np.argmax(res_score < threshold)] for res_label, res_score in zip(res_labels, res_scores)))
top_res_labels
all_labels = np.concatenate([train_labels, test_labels], axis=0)
labels = np.array(list(set(label for label_list in all_labels for label in label_list)))

label2idx = {label: idx for idx, label in enumerate(labels)}
idx2label = {idx: label for idx, label in enumerate(labels)}
mlb = MultiLabelBinarizer(sparse_output=False, classes=labels)
res_sparse = mlb.fit_transform(top_res_labels)
test_sparse = mlb.fit_transform(test_labels)
MCM = multilabel_confusion_matrix(test_sparse, res_sparse, labels=[label2idx[label] for label in labels])
MCM.shape
# a confusion matrix
# [[TN FN]
#  [FP TP]]

print("CONFUSION MATRIX FORMAT:\n[[true_negatives  false_negatives]\n [false_positives true_positives ]]")
for idx, matrix in enumerate(MCM):
    print(idx2label[idx])
    print(matrix)<jupyter_output>CONFUSION MATRIX FORMAT:
[[true_negatives  false_negatives]
 [false_positives true_positives ]]
protect_from_non-living_threats
[[46 35]
 [ 4 15]]
manage_environmental_disturbances
[[98  0]
 [ 2  0]]
prevent_fatigue
[[100   0]
 [  0   0]]
sense_disease
[[100   0]
 [  0   0]]
protect_from_loss_of_liquids
[[96  0]
 [ 4  0]]
modify/convert_mechanical_energy
[[99  0]
 [ 1  0]]
passively_move_through/on_liquids
[[97  0]
 [ 3  0]]
sense_shape_and/or_pattern
[[98  0]
 [ 2  0]]
send_signals
[[91  5]
 [ 2  2]]
store_resources
[[96  0]
 [ 4  0]]
send_light_signals_in_the_non-visible_spectrum
[[100   0]
 [  0   0]]
protect_from_harm
[[33 35]
 [ 7 25]]
distribute_solids
[[97  0]
 [ 3  0]]
chemically_modify_or_change_energy_state
[[82 12]
 [ 5  1]]
store_energy
[[99  0]
 [ 1  0]]
protect_from_living_threats
[[80  5]
 [10  5]]
expel_liquids
[[100   0]
 [  0   0]]
manage_environmental_disturbances_in_a_community
[[100   0]
 [  0   0]]
actively_move_through/on_solids
[[96  1]
 [ 3  0]]
protect_from_an[...]<jupyter_text><jupyter_code>confmat_data = [(idx, idx2label[idx], matrix[0][0], matrix[0][1], matrix[1][0], matrix[1][1]) for idx, matrix in enumerate(MCM)]
columns = ['index', 'label', 'TN', 'FN', 'FP', 'TP']
confmat_df = pd.DataFrame(confmat_data, columns=columns)
confmat_df
confmat_df[['precision']] = confmat_df.apply(
    lambda row: 0 if (row['TP'] + row['FP']) == 0 else (row['TP'] / (row['TP'] + row['FP'])),
    axis=1
)<jupyter_output><empty_output><jupyter_text>New confusion matrix work<jupyter_code># top_res_labels = res_labels[:,:k]
threshold = 0.1
# np.argmax(res_score < threshold) becomes the number of labels scoring above the threshold
top_res_labels = np.array(list(res_label[:np.argmax(res_score < threshold)] for res_label, res_score in zip(res_labels, res_scores)))
top_res_labels
parent_labels = set()
with open(f'{DATASET}/taxonomy.txt', 'r') as tax_file:
    for line in tax_file:
        parent_label = line.split()[0]
        parent_labels.add(parent_label)
len(parent_labels)
all_labels = np.concatenate([train_labels, test_labels], axis=0)

label_count = dict()
for label in label2idx:
    label_count[label] = 0

for label_list in all_labels:
    for label in label_list:
        label_count[label] += 1

labels = np.array(sorted(list(set(label for label_list in all_labels for label in label_list)), 
                         key=lambda lbl: label_count[lbl],
                         reverse=True))

label2idx = {label: idx for idx, label in enumerate(labels)}
idx2label = {idx: label for idx, label in enumerate(labels)}
preds_for_test_label = dict()
test_label_count = dict()

for test_label in label2idx:
    test_label_count[test_label] = 0

for test_label in label2idx:
    preds_for_test_label[test_label] = dict()
    for res_label in label2idx:
        preds_for_test_label[test_label][res_label] = 0

for test_label_list, res_label_list, res_score_list in zip(test_labels, res_labels, res_scores):
    for test_label in test_label_list:
        test_label_count[test_label] += 1
        for res_label, res_score in zip(res_label_list, res_score_list):
            preds_for_test_label[test_label][res_label] += res_score

    # print("test_label_list", test_label_list)
    # print("top_res_label_list", top_res_label_list)
num_labels = len(label2idx)
conf_matrix = np.array(
    [
     [
      # i is label2idx[test_label], j is label2idx[res_label]
      preds_for_test_label[idx2label[i]][idx2label[j]] / test_label_count[idx2label[i]] if test_label_count[idx2label[i]] > 0 else 0
      for j in range(num_labels)
     ]
     for i in range(num_labels)
    ]
    
)

conf_matrix
rc('xtick', labelsize=7)
rc('ytick', labelsize=7)
rc('font', size=30)

label_limit = 131
conf_matrix_small = conf_matrix[:label_limit, :label_limit]
num_labels = label_limit
row_labels = [idx2label[i] for i in range(num_labels)]
col_labels = [idx2label[i] for i in range(num_labels)]

plt.rcParams["figure.figsize"] = (15, 15)
fig, ax = plt.subplots()
plt.matshow(conf_matrix_small, fignum=0)
ax.set_title('Multilabel Confusion Matrix for MATCH on PeTaL data\nLabels Sorted by Frequency', y=1.0, pad=15)
ax.set_xlabel('Predicted labels')
ax.set_ylabel('Ground truth labels')
ax.xaxis.set_label_position('top')
ax.xaxis.tick_top()
plt.xticks(range(num_labels), col_labels, rotation='vertical')
plt.yticks(range(num_labels), row_labels)
plt.colorbar()
# plt.rcParams["axes.titley"] = 1.0
# plt.rcParams["axes.titlepad"] = 15
plt.savefig(fname='../plots/mcm_20210721.jpg', facecolor='w', transparent=False)

from matplotlib import pyplot as plt
matplotlib.rcParams.keys()
len(idx2label)<jupyter_output><empty_output><jupyter_text>Only Leaf Labels<jupyter_code>all_labels = np.concatenate([train_labels, test_labels], axis=0)

label_count = dict()
for label in label2idx:
    label_count[label] = 0

for label_list in all_labels:
    for label in label_list:
        label_count[label] += 1

labels = np.array(sorted(filter(lambda lbl: not lbl in parent_labels, 
                                list(set(label for label_list in all_labels for label in label_list))
                                ), 
                         key=lambda lbl: label_count[lbl],
                         reverse=True))

label2idx = {label: idx for idx, label in enumerate(labels)}
idx2label = {idx: label for idx, label in enumerate(labels)}
preds_for_test_label = dict()
test_label_count = dict()

for test_label in label2idx:
    test_label_count[test_label] = 0

for test_label in label2idx:
    preds_for_test_label[test_label] = dict()
    for res_label in label2idx:
        preds_for_test_label[test_label][res_label] = 0

for test_label_list, res_label_list, res_score_list in zip(test_labels, res_labels, res_scores):
    for test_label in test_label_list:
        if test_label in test_label_count:
            test_label_count[test_label] += 1
            for res_label, res_score in zip(res_label_list, res_score_list):
                if res_label in preds_for_test_label[test_label]:
                    preds_for_test_label[test_label][res_label] += res_score

    # print("test_label_list", test_label_list)
    # print("top_res_label_list", top_res_label_list)
num_labels = len(label2idx)
conf_matrix = np.array(
    [
     [
      # i is label2idx[test_label], j is label2idx[res_label]
      preds_for_test_label[idx2label[i]][idx2label[j]] / test_label_count[idx2label[i]] if test_label_count[idx2label[i]] > 0 else 0
      for j in range(num_labels)
     ]
     for i in range(num_labels)
    ]
    
)

conf_matrix
rc('xtick', labelsize=12)
rc('ytick', labelsize=12)
rc('font', size=30)

label_limit = 20
conf_matrix_small = conf_matrix[:label_limit, :label_limit]
num_labels = label_limit
row_labels = [idx2label[i] for i in range(num_labels)]
col_labels = [idx2label[i] for i in range(num_labels)]

plt.rcParams["figure.figsize"] = (15, 15)
fig, ax = plt.subplots()
plt.matshow(conf_matrix_small, fignum=0)
ax.set_title('Multilabel Confusion Matrix for MATCH on PeTaL data\nTop 20 Leaf Labels Sorted by Frequency', y=1.0, pad=15)
ax.set_xlabel('Predicted labels')
ax.set_ylabel('Ground truth labels')
ax.xaxis.set_label_position('top')
ax.xaxis.tick_top()
plt.xticks(range(num_labels), col_labels, rotation='vertical')
plt.yticks(range(num_labels), row_labels)
plt.colorbar()
# plt.rcParams["axes.titley"] = 1.0
# plt.rcParams["axes.titlepad"] = 15
plt.savefig(fname='../plots/mcm_20210721.jpg', facecolor='w', transparent=False)<jupyter_output><empty_output><jupyter_text>Previous work (ignore)<jupyter_code>!wc predictions.txt
!wc MAG/test_labels.txt
!head -n 50 predictions.txt
print("\n")
!head -n 50 MAG/test_labels.txt
pred_path = "predictions.txt"
test_path = "MAG/test_labels.txt"
# TO DO: compute precision, recall, F1 score, confusion matrix
rows_list = []
with open(pred_path, 'r') as pred_fn, open(test_path, 'r') as test_fn:
    for pred_line, test_line in zip(pred_fn, test_fn):
        pred_labels = [int(label) for label in pred_line.strip().split()]
        test_labels = [int(label) for label in test_line.strip().split()]
        dict1 = {'pred_labels': pred_labels, 'test_labels': test_labels}
        rows_list.append(dict1)
label_df = pd.DataFrame(rows_list)
label_df
label_df['pred_labels'][1]
def calc_precision():
    tp = 0
    tp_fp = 0
    for index, row in label_df.iterrows():
        pred_labels, test_labels = row['pred_labels'], row['test_labels']
        tp_fp += 1
        if pred_labels[0] in test_labels:
            tp += 1
        # for pred_label in pred_labels:
        #     if pred_label in test_labels:
        #         true_positives += 1
    return tp / tp_fp

print(f"Precision @ 1: {calc_precision()}")
def calc_recall():
    tp = 0
    tp_fn = 0
    for index, row in label_df.iterrows():
        pred_labels, test_labels = row['pred_labels'], row['test_labels']
        for test_label in test_labels:
            tp_fn += 1
            if test_label in pred_labels:
                tp += 1
    return tp / tp_fn
print(f"Recall @ 5: {calc_recall()}")<jupyter_output><empty_output><jupyter_text>Custom calculation of precision and recall.<jupyter_code>res_labels = np.load(f"{DATASET}/results/{MODEL}-{DATASET}-labels.npy")
res_scores = np.load(f"{DATASET}/results/{MODEL}-{DATASET}-scores.npy")
test_labels = np.load(f"{DATASET}/test_labels.npy")
Stats = namedtuple("Stats", "threshold topk precision recall f1")

def compute_stats(threshold):
    precisions = []
    recalls = []
    topks = []
    f1s = []
    for res_label, res_score, test_label in zip(res_labels, res_scores, test_labels):
        topk = np.argmax(res_score < threshold) # topk becomes the number of labels scoring above the threshold
        precision = 1.0 if topk == 0 else np.mean([1 if x in test_label else 0 for x in res_label[:topk]])
        recall = np.mean([1 if x in res_label[:topk] else 0 for x in test_label])
        f1 = 0 if (precision + recall) == 0 else (2 * precision * recall) / (precision + recall)
        topks.append(topk)
        precisions.append(precision)
        recalls.append(recall)
        f1s.append(f1)
        # print(res_label[:topk], precision, recall)
    return Stats(threshold, np.mean(topks), np.mean(precisions), np.mean(recalls), np.mean(f1s))
    # print("Threshold:", threshold)
    # print("Average number of labels:", np.mean(topks))
    # print("Average precision:", np.mean(precisions))
    # print("Average recall:", np.mean(recalls))
    # print("Average F1 score:", np.mean(f1s))
compute_stats(0.5)
ps = []
rs = []
ts = []
f1s = []
for i in tqdm(range(1, 10000)):
    threshold = i / 10000
    stats = compute_stats(threshold)
    ps.append(stats.precision)
    rs.append(stats.recall)
    ts.append(threshold)
    f1s.append(stats.f1)
plt.plot(ps, rs, linestyle='--')
plt.grid()
plt.title('Precision-Recall Curve for MATCH on cleaned_lens_output.json')
plt.xlabel('Recall')
plt.xlim(0, 1)
plt.ylabel('Precision')
plt.ylim(0, 1)
plt.grid()
plt.title('Precision-Recall Curve for MATCH on cleaned_lens_output.json')
plt.plot(ts, ps, linestyle='--', label='Precision')
plt.plot(ts, rs, linestyle='--', label='Recall')
plt.plot(ts, f1s, linestyle='--', label='F1 score')
plt.xlabel('Threshold')
plt.xlim(0, 1)
plt.ylabel('Metrics')
plt.ylim(0, 1)
plt.legend()<jupyter_output><empty_output><jupyter_text>Compute similar statistics for MAG-CS dataset<jupyter_code>DATASET = 'MAG'

res_labels = np.load(f"{DATASET}/results/{MODEL}-{DATASET}-labels.npy")
res_scores = np.load(f"{DATASET}/results/{MODEL}-{DATASET}-scores.npy")
test_labels = np.load(f"{DATASET}/test_labels.npy")
thresholds = list(x / 1000 for x in range(1, 10)) + \
    list(x / 100 for x in range(1, 10)) + \
    list(x / 10 for x in range(1, 10)) + \
    list((90 + x) / 100 for x in range(1, 10)) + \
    list((990 + x) / 1000 for x in range(1, 10))
ps = []
rs = []
ts = []
f1s = []
for i in tqdm(thresholds):
    threshold = i / 100
    stats = compute_stats(threshold)
    ps.append(stats.precision)
    rs.append(stats.recall)
    ts.append(threshold)
    f1s.append(stats.f1)
plt.plot(ps, rs, linestyle='--')
plt.grid()
plt.title('Precision-Recall Curve for MATCH on MAG-CS')
plt.xlabel('Recall')
plt.xlim(0, 1)
plt.ylabel('Precision')
plt.ylim(0, 1)
plt.grid()
plt.title('Precision-Recall Curve for MATCH on MAG-CS')
plt.plot(ts, ps, linestyle='--', label='Precision')
plt.plot(ts, rs, linestyle='--', label='Recall')
plt.plot(ts, f1s, linestyle='--', label='F1 score')
plt.xlabel('Threshold')
plt.xlim(0, 1)
plt.ylabel('Metrics')
plt.ylim(0, 1)
plt.legend()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup