<jupyter_start><jupyter_text><jupyter_code># load in the testset
# original res testing result:  test_match : 0.9156 test_hs : 0.8540
if not os.path.exists('task.pickle'):
  task_url = os.path.join(data_dir, 'task')
  seq_task, task_labels, _ = dataset(task_url, monitor=False)
  vocab = pickle.load(open(vocab_url, 'rb'))
  vocab = list(zip(*sorted(vocab.items(), key=lambda x: x[1])))[0]
  task_text = []
  for idx_list in seq_task:
      s = ""
      for idx in idx_list:
          s += vocab[idx] + " " 
      task_text.append(s)
  task = {'text': task_text,
          'id': np.arange(len(task_text)),
          'labels': task_labels}
  with open('task.pickle', 'wb') as handle:
      pickle.dump(task, handle, protocol=pickle.HIGHEST_PROTOCOL)
task_ds = reader.read('task.pickle')


'''
trainer.set_validation(task_ds)
prob, _ = trainer.predict()
tmp = trainer.val_y.detach().cpu().numpy()
pred = prob > 0.5                
test_match = np.mean([(pred[i][tmp[i]==1]==1).any() for i in range(len(pred))])
test_hs = (((pred==1)*(tmp==1)).sum(1)/(((pred==1)+(tmp==1))>0).sum(1)).mean()

print("testing result: ",
    "test_match : %0.4f" % test_match,
    "test_hs : %0.4f" % test_hs
  ) 
'''

# seems the original code (code used when doing this experiments) doesn't save the final model to trainer.model
# so here we manually load the model in trainer and do evaluation here
# best testing result:  test_match : 0.9162 test_hs : 0.8686

from allennlp.data.iterators import BucketIterator
iterator = BucketIterator(batch_size=30, sorting_keys=[("tokens", "num_tokens")])
iterator.index_with(vocab)
model = Leam_Classifier(len(class_names), 256, 256, 10, 
                            n_layer=1, dropout_rate=0.5, embpath=None, label_att=True, multilabel=True)
model.load_state_dict(torch.load('model.pth'))
trainer = Trainer(iterator, batch_size=30, num_epoches=50, learning_rate=3e-4, valid_freq=10, model_type='embed')

trainer.init_model(model)
trainer.set_validation(task_ds)
prob, _ = trainer.predict()
tmp = trainer.val_y.detach().cpu().numpy()
pred = prob > 0.5                
test_match = np.mean([(pred[i][tmp[i]==1]==1).any() for i in range(len(pred))])
test_hs = (((pred==1)*(tmp==1)).sum(1)/(((pred==1)+(tmp==1))>0).sum(1)).mean()

precision_list = []
recall_list = []
for i in range(101):
    cur_thresh = i * 0.01
    pred = prob > cur_thresh
    precision = (((pred==1)*(tmp==1)).sum(1) / ((pred==1)).sum(1)).mean()
    recall =  (((pred==1)*(tmp==1)).sum(1) / ((tmp==1)).sum(1)).mean()
    precision_list.append(precision)
    recall_list.append(recall)
    

print("testing result: ",
    "test_match : %0.4f" % test_match,
    "test_hs : %0.4f" % test_hs
  ) 


precision_list = []
recall_list = []
fp_list = []
for i in range(0, 10001):
    cur_thresh = i * 0.0001
    pred = (prob > cur_thresh).astype(np.int)
    precision = ((pred * tmp).sum(1) / (pred.sum(1) + 1e-5)).mean()
    recall =  ((pred * tmp).sum(1) / (tmp.sum(1) + 1e-5)).mean()
    fp = 1 - ((((pred == 0) * (tmp ==0)).sum(1)) / ((tmp==0).sum(1) + 1e-5)).mean()
    precision_list.append(precision)
    recall_list.append(recall)
    fp_list.append(fp)
#print(precision_list)
#print(recall_list)
from sklearn.metrics import auc
from matplotlib import pyplot as plt
myauc = auc(recall_list, precision_list)
print(myauc)
plt.figure()
plt.plot(recall_list, precision_list)
plt.ylabel('precision')
plt.xlabel('recall')
plt.title('precision recall curve')

myauc= auc(fp_list, recall_list)
print(myauc)
plt.figure()
plt.plot(fp_list, recall_list)
plt.ylabel('recall')
plt.xlabel('false positive rate')
plt.title('ROC curve')


torch.save(model.state_dict(), 'model.pth')
# here we try to use ELMO without LEAM
from allennlp.data.iterators import BucketIterator
iterator = BucketIterator(batch_size=30, sorting_keys=[("tokens", "num_tokens")])
iterator.index_with(vocab)
model = Leam_Classifier(len(class_names), 256, 256, 10, 
                            n_layer=1, dropout_rate=0.5, embpath=None, label_att=False, multilabel=True)
#model.load_state_dict(torch.load('model1.pth'))
trainer = Trainer(iterator, batch_size=30, num_epoches=120, learning_rate=1e-3, valid_freq=10, model_type='noembed')
trainer.set_validation(test_ds)
trainer.init_model(model)
train_prob = trainer.fit(train_ds, class_names)
torch.save(trainer.model.state_dict(), 'model_no_LEAM.pth')
prob, _ = trainer.predict()
tmp = trainer.val_y.detach().cpu().numpy()
pred = prob > 0.5                
val_match = np.mean([(pred[i][tmp[i]==1]==1).any() for i in range(len(pred))])
val_hs = (((pred==1)*(tmp==1)).sum(1)/(((pred==1)+(tmp==1))>0).sum(1)).mean()

print("Validation result: ",
    "val_match : %0.4f" % val_match,
    "val_hs : %0.4f" % val_hs
  ) 
task_ds = reader.read('task.pickle')
trainer.set_validation(task_ds)
prob, _ = trainer.predict()
tmp = trainer.val_y.detach().cpu().numpy()
pred = prob > 0.5                
test_match = np.mean([(pred[i][tmp[i]==1]==1).any() for i in range(len(pred))])
test_hs = (((pred==1)*(tmp==1)).sum(1)/(((pred==1)+(tmp==1))>0).sum(1)).mean()

print("testing result: ",
    "test_match : %0.4f" % test_match,
    "test_hs : %0.4f" % test_hs
  )<jupyter_output>3019it [00:01, 2525.56it/s]
StarcoderdataJupyterStructuredCleanDedup