<jupyter_start><jupyter_text>Training Roberta for relation classification[Code based on this](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta).Our goal is to train a binary classification model to determine if drug-treatment sentences actually contain a relation.<jupyter_code>!pip install transformers
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
import tensorflow.keras.backend as K
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import EarlyStopping
# data was made with the make-train notebook also found in the repo.
!gsutil cp  gs://coronaviruspublicdata/snapshot_re_4_12_2020/drug_train_data.csv drug_train_data.csv
!gsutil cp  gs://coronaviruspublicdata/snapshot_re_4_12_2020/drug_dev_data.csv drug_dev_data.csv 
!gsutil cp  gs://coronaviruspublicdata/snapshot_re_4_12_2020/drug_test_data.csv drug_test_data.csv
!gsutil cp  gs://coronaviruspublicdata/snapshot_re_4_12_2020/synth_train_data.csv synth_train_data.csv
!gsutil cp  gs://coronaviruspublicdata/snapshot_re_4_12_2020/synth_dev_data.csv synth_dev_data.csv 
!gsutil cp  gs://coronaviruspublicdata/snapshot_re_4_12_2020/synth_test_data.csv synth_test_data.csv
def create_train_dev_test(prefix):
  train = pd.read_csv(prefix + "_train_data.csv")
  dev = pd.read_csv(prefix + "_dev_data.csv")
  test = pd.read_csv(prefix + "_test_data.csv")
  return train, test, dev

transfer_train, transfer_test, transfer_dev = create_train_dev_test("synth")
drug_train, drug_test, drug_dev = create_train_dev_test("drug")
drug_full = pd.concat([drug_train, drug_test])

def regular_encode(texts, tokenizer, maxlen=512):
    """ Function to encode many sentences"""
    enc_di = tokenizer.batch_encode_plus(
        texts, 
        return_attention_masks=False, 
        return_token_type_ids=False,
        pad_to_max_length=True,
        max_length=maxlen
    )
    return np.array(enc_di['input_ids'])
def build_model(transformer, max_len=512):
    """
    Require a transformer of type TFAutoBert
    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras
    """
    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
    sequence_output = transformer(input_word_ids)[0]
    cls_token = sequence_output[:, 0, :]
    out = Dense(1, activation='sigmoid')(cls_token)
    
    model = Model(inputs=input_word_ids, outputs=out)
    model.compile(Adam(lr=1e-6), loss='binary_crossentropy', metrics=[tf.keras.metrics.Recall(), tf.keras.metrics.Precision(), 'accuracy'])
    return model
max_len = int(int(drug_train.sentence.str.len().max()))
avg_len = int(drug_train.sentence.str.len().median())

# Configuration params
EPOCHS = 4
MAX_LEN = 512
MODEL = "allenai/biomed_roberta_base" 

# First load the real tokenizer
tokenizer = AutoTokenizer.from_pretrained(MODEL)
%%time 
x_train = regular_encode(transfer_train.sentence.values, tokenizer, maxlen=MAX_LEN)
x_valid = regular_encode(transfer_dev.sentence.values, tokenizer, maxlen=MAX_LEN)
x_test = regular_encode(transfer_test.sentence.values, tokenizer, maxlen=MAX_LEN)
y_train = transfer_train.label.values
y_valid = transfer_dev.label.values
y_test  = transfer_test.label.values
es = EarlyStopping(monitor='val_accuracy', 
                    min_delta=0.001, 
                    patience=3,
                    verbose=1, 
                    mode='max', 
                    restore_best_weights=True)
!pip install wandb
!wandb login
import wandb
from wandb.keras import WandbCallback
wandb.init(project="vt-relation-extract", sync_tensorboard=True)
from transformers import AutoModel
strategy = tf.distribute.MirroredStrategy(cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())
model = AutoModel.from_pretrained("allenai/biomed_roberta_base")
!mkdir biomed_roberta_base
model.save_pretrained("biomed_roberta_base")
with strategy.scope():
  model = TFAutoModel.from_pretrained("biomed_roberta_base", from_pt=True)
  model = build_model(model)
BATCH_SIZE = 2 * strategy.num_replicas_in_sync

model.summary()
# This may look like a bug but in reality we only care about the performance on
# the annotated drug data and not what we are training on.
x_test = regular_encode(drug_full.sentence.values, tokenizer, maxlen=MAX_LEN)
y_test  = drug_full.label.values

train_history = model.fit(
                        x_train, y_train,
                        batch_size = BATCH_SIZE,
                        validation_data=(x_test, y_test),
                        callbacks=[es, WandbCallback()],
                        epochs=EPOCHS
                        )
x_train = regular_encode(drug_train.sentence.values, tokenizer, maxlen=MAX_LEN)
x_valid = regular_encode(drug_dev.sentence.values, tokenizer, maxlen=MAX_LEN)
x_test = regular_encode(drug_test.sentence.values, tokenizer, maxlen=MAX_LEN)
y_train = drug_train.label.values
y_valid = drug_dev.label.values
y_test  = drug_test.label.values
train_history = model.fit(
                        x_train, y_train,
                        batch_size=BATCH_SIZE,
                        validation_data=(x_test, y_test),
                        callbacks=[es, WandbCallback()],
                        epochs=8
                        )<jupyter_output>Epoch 1/8
WARNING:tensorflow:Gradients do not exist for variables ['tf_roberta_model_4/roberta/pooler/dense/kernel:0', 'tf_roberta_model_4/roberta/pooler/dense/bias:0'] when minimizing the loss.
WARNING:tensorflow:Gradients do not exist for variables ['tf_roberta_model_4/roberta/pooler/dense/kernel:0', 'tf_roberta_model_4/roberta/pooler/dense/bias:0'] when minimizing the loss.
  2/100 [..............................] - ETA: 7s - precision_4: 1.0000 - accuracy: 1.0000 - loss: 0.1115 - recall_4: 1.0000WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.265747). Check your callbacks.
100/100 [==============================] - ETA: 0s - precision_4: 0.7542 - accuracy: 0.7800 - loss: 0.4871 - recall_4: 0.8558<jupyter_text>Saving/ExportingA model isn't useful if it cannot be used in a production pipeline.<jupyter_code>from google.colab import auth
from datetime import datetime
auth.authenticate_user()
!gsutil cp -r best_epoch_roberta gs://coronaviruspublicdata/temp_data/snapshots
import pickle 
pickle.dump(model, open( "model.pickle", "wb" ) )
!gsutil cp model.pickle gs://coronaviruspublicdata/model.pickle
import pickle
def save_model(model, transformer_dir='transformer3'):
    """
    Special function to load a keras model that uses a transformer layer
    """
    transformer = model.layers[1]
    transformer.save_pretrained(transformer_dir)
    sigmoid = model.get_layer(index=3).get_weights()
    pickle.dump(sigmoid, open('sigmoid3.pickle', 'wb'))

def load_model(transformer_dir='transformer3', max_len=256):
    """
    Special function to load a keras model that uses a transformer layer
    """
    transformer = TFAutoModel.from_pretrained(transformer_dir)
    model = build_model(transformer, max_len=max_len)
    sigmoid = pickle.load(open('sigmoid3.pickle', 'rb'))
    model.get_layer('sigmoid').set_weights(sigmoid)

    return model
!mkdir transformer3
save_model(model)

model.summary()
model.get_layer(index=3)
!gsutil cp -r transformer3 gs://coronaviruspublicdata/re_final_best2/s
!gsutil cp sigmoid3.pickle gs://coronaviruspublicdata/re_final_best2/s<jupyter_output>Copying file://transformer3/tf_model.h5 [Content-Type=application/octet-stream]...
/ [0 files][    0.0 B/475.7 MiB]                                                
==> NOTE: You are uploading one or more large file(s), which would run
significantly faster if you enable parallel composite uploads. This
feature can be enabled by editing the
"parallel_composite_upload_threshold" value in your .boto
configuration file. However, note that if you do this large files will
be uploaded as `composite objects
<https://cloud.google.com/storage/docs/composite-objects>`_,which
means that any user who downloads such objects will need to have a
compiled crcmod installed (see "gsutil help crcmod"). This is because
without a compiled crcmod, computing checksums on composite objects is
so slow that gsutil disables downloads of composite objects.

Copying file://transformer3/config.json [Content-Type=application/json]...
\ [2 files][475.7 MiB/475.7 MiB]                                                
Oper[...]<jupyter_text>Qualitative EvaluationWe will now qualitatively look at a few examples.<jupyter_code>test_examples = regular_encode(["As with Acacia and PAN, the LAC prospectus affirms that digital technologies and ICTbased solutions provide a powerful tool to change the ways in which health services are managed and delivered to the population at large, and to low-income and marginalized communities in particular"], tokenizer, maxlen=MAX_LEN)
model.predict(test_examples)
test_examples = regular_encode(["Glatiramer acetate (Copaxone) therapy induces an oligoclonal CD8+ T cell response with cytotoxic ability R"], tokenizer, maxlen=MAX_LEN)
model.predict(test_examples)
predictions = model.predict(x_valid)
drug_dev.sentence.values[0]
drug_dev.label.values[0]
print(predictions[0])
print("sentence is " + drug_dev.sentence.values[0])
print("real label is " + str(drug_dev.label.values[0]))
print(predictions[2])
print("sentence is: " + drug_dev.sentence.values[2])
print("real label is " + str(drug_dev.label.values[2]))
print(predictions[4])
print("sentence is: " + drug_dev.sentence.values[4])
print("real label is " + str(drug_dev.label.values[4]))<jupyter_output>[0.6915216]
sentence is: The remaining five successful patients had mild reactions which were treated symptomatically with acetaminophen, antihistamines or both
real label is 1<jupyter_text>Tests for RAM usageBasic check to determine how much RAM is available.<jupyter_code>!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi
!pip install gputil
!pip install psutil
!pip install humanize
import psutil
import humanize
import os
import GPUtil as GPU
GPUs = GPU.getGPUs()
# XXX: only one GPU on Colab and isn’t guaranteed
gpu = GPUs[0]
def printm():
 process = psutil.Process(os.getpid())
 print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), " | Proc size: " + humanize.naturalsize( process.memory_info().rss))
 print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))
printm()
from google.colab import auth
from datetime import datetime
auth.authenticate_user()

!gsutil cp -r transformer gs://coronaviruspublicdata/re_snapshot/4_13_2020
!gsutil cp sigmoid.pickle gs://coronaviruspublicdata/re_snapshot/4_13_2020<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup