<jupyter_start><jupyter_text>Setup<jupyter_code># pip install pytorch_pretrained_bert pytorch-nlp
import sys
import numpy as np
import random as rn
import torch
from pytorch_pretrained_bert import BertModel
from torch import nn
from pytorch_pretrained_bert import BertTokenizer
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.optim import Adam
from torch.nn.utils import clip_grad_norm_
from IPython.display import clear_output
rn.seed(10)
np.random.seed(10)
torch.manual_seed(10)
torch.cuda.manual_seed(10)<jupyter_output><empty_output><jupyter_text>News Aggregator Dataset<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import keras
import tensorflow as tf

# keras: for data processing
from keras.preprocessing.sequence import pad_sequences
from keras.utils.np_utils import to_categorical
# data load
data = pd.read_csv('./data/uci_news_aggregator.csv', delimiter = ',', skiprows = 1,
                   names = ['ID', 'TITLE', 'URL', 'PUBLISHER', 'CATEGORY', 'STORY',
                            'HOSTNAME', 'TIMESTAMP'], usecols=['TITLE', 'CATEGORY'])
data = data.reindex(np.random.permutation(data.index))
data.shape
data.head(20)
# data slicing
num_of_categories = 12500
shuffled = data.reindex(np.random.permutation(data.index))

e = shuffled[shuffled['CATEGORY'] == 'e']#[:num_of_categories]
b = shuffled[shuffled['CATEGORY'] == 'b']#[:num_of_categories]
t = shuffled[shuffled['CATEGORY'] == 't']#[:num_of_categories]
m = shuffled[shuffled['CATEGORY'] == 'm']#[:num_of_categories]

concated = pd.concat([e,b,t,m], ignore_index=True)
# label col
concated['LABEL'] = 0
np.random.seed(41)
concated = concated.reindex(np.random.permutation(concated.index))

concated.loc[concated['CATEGORY'] == 'e', 'LABEL'] = 0
concated.loc[concated['CATEGORY'] == 'b', 'LABEL'] = 1
concated.loc[concated['CATEGORY'] == 't', 'LABEL'] = 2
concated.loc[concated['CATEGORY'] == 'm', 'LABEL'] = 3
concated['LABEL'][:10]
# one-hot encoding
labels = to_categorical(concated['LABEL'], num_classes=4)
labels<jupyter_output><empty_output><jupyter_text>BERT Tokenizer<jupyter_code>word_max_len = 64
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
 concated['TITLE'].values[:3]
tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t) + ['[SEP]'], concated['TITLE'].values))
tokens[:3]
tokens_ids = pad_sequences(list(map(tokenizer.convert_tokens_to_ids, tokens)), maxlen=word_max_len, truncating="post", padding="post", dtype="int")
tokens_ids.shape
tokens_ids[:1]
masks = [[float(i > 0) for i in ii] for ii in tokens_ids]
masks[:1]<jupyter_output><empty_output><jupyter_text>BERT Baseline<jupyter_code>class BertClassifier(nn.Module):
    def __init__(self, dropout=0.1):
        super(BertClassifier, self).__init__()

        self.bert = BertModel.from_pretrained('bert-base-uncased')

        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, 4)
        self.sigmoid = nn.Sigmoid()
        self.softmax = nn.Softmax()
    
    def forward(self, tokens, masks=None):
        _, pooled_output = self.bert(tokens, attention_mask=masks, output_all_encoded_layers=False)
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
#         proba = self.sigmoid(linear_output)
        proba = self.softmax(linear_output)
        return proba
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device
str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'
bert_clf = BertClassifier()
bert_clf = bert_clf.cuda()
str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'
x = torch.tensor(tokens_ids[:3]).to(device)
y, pooled = bert_clf.bert(x, output_all_encoded_layers=False)
x.shape, y.shape, pooled.shape
x
y
pooled
y = bert_clf(x)
y.cpu().detach().numpy()
str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'
y, x, pooled = None, None, None
torch.cuda.empty_cache()
str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'<jupyter_output><empty_output><jupyter_text>Fine-tune BERT - Train / Test<jupyter_code>BATCH_SIZE = 3
EPOCHS = 3
split_size = 4 * int(len(tokens_ids) / 5)
X_train_ids, X_test_ids = tokens_ids[:split_size,:], tokens_ids[split_size:,:]
tokens_ids.shape
X_train_ids.shape
X_test_ids.shape
y_train, y_test = labels[:split_size,:], labels[split_size:,:]
labels.shape
y_train.shape
y_test.shape
masks_train, masks_test = np.array(masks)[:split_size,:], np.array(masks)[split_size:,:]
np.array(masks).shape
masks_train.shape
masks_test.shape
train_tokens_tensor = torch.tensor(X_train_ids)
train_y_tensor = torch.tensor(y_train).float()

test_tokens_tensor = torch.tensor(X_test_ids)
test_y_tensor = torch.tensor(y_test).float()

train_masks_tensor = torch.tensor(masks_train)
test_masks_tensor = torch.tensor(masks_test)
str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'
train_dataset = TensorDataset(train_tokens_tensor, train_masks_tensor, train_y_tensor)
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)

test_dataset = TensorDataset(test_tokens_tensor, test_masks_tensor, test_y_tensor)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)
len(train_dataloader)
len(test_dataloader)
train_dataset.tensors<jupyter_output><empty_output><jupyter_text>- Fine-tune BERT<jupyter_code>optimizer = Adam(bert_clf.parameters(), lr=3e-6)
torch.cuda.empty_cache()
for epoch_num in range(EPOCHS):
    bert_clf.train()
    train_loss = 0
    for step_num, batch_data in enumerate(train_dataloader):
        
        token_ids, masks, labels = tuple(t.to(device) for t in batch_data)
        logits = bert_clf(token_ids, masks)
        
        loss_func = nn.BCELoss()

        batch_loss = loss_func(logits, labels)
        train_loss += batch_loss.item()
        
        bert_clf.zero_grad()
        batch_loss.backward()

        clip_grad_norm_(parameters=bert_clf.parameters(), max_norm=1.0)
        optimizer.step()
        
        clear_output(wait=True)
        print('Epoch: ', epoch_num + 1)
        print("\r" + "{0}/{1} loss: {2} ".format(step_num, int(len(X_train_ids) / BATCH_SIZE), train_loss / (step_num + 1)))
torch.save(bert_clf, './bert_clf_' + str(EPOCHS) + 'epoch' + '.h5')<jupyter_output>/home/taeyong/anaconda3/lib/python3.6/site-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type BertClassifier. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "<jupyter_text>Evaluation<jupyter_code>bert_clf.eval()
bert_predicted = []
all_logits = []
with torch.no_grad():
    for step_num, batch_data in enumerate(test_dataloader):

        token_ids, masks, labels = tuple(t.to(device) for t in batch_data)
        logits = bert_clf(token_ids, masks)
        
        loss_func = nn.BCELoss()
        
        loss = loss_func(logits, labels)
        numpy_logits = logits.cpu().detach().numpy()
        
        for i in range(len(logits)):
             bert_predicted.append(logits[i].argmax())
#         bert_predicted += list(numpy_logits[:, 0] > 0.5)
        all_logits += list(numpy_logits[:, 0])
        
        clear_output(wait=True)
        print("\r" + "{0}/{1} loss: {2} ".format(step_num, int(len(X_test_ids) / BATCH_SIZE), train_loss / (step_num + 1)))
bert_predicted = np.array(bert_predicted)
len(bert_predicted)
bert_predicted
y_test<jupyter_output><empty_output><jupyter_text>- Accuracy of classification<jupyter_code>correct_count = 0
for i in range(len(bert_predicted)):
    y = y_test[i].argmax()
    
    if bert_predicted[i].item() == y:
        correct_count = correct_count + 1
correct_count / len(bert_predicted)<jupyter_output><empty_output><jupyter_text>- Confusion matrix<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    fig, ax = plt.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    return ax
class_names = np.array(['B', 'E', 'T', 'M'])
confusion_matrix_predicted = [element.item() for element in bert_predicted.flatten()]
confusion_matrix_predicted
y_test.argmax(axis=1)
# Plot normalized confusion matrix
plot_confusion_matrix(y_test.argmax(axis=1), confusion_matrix_predicted, classes=class_names, normalize=True,
                      title='Confusion Matrix')

plt.show()<jupyter_output>Normalized confusion matrix
[[0.98230992 0.00882857 0.0065226  0.00233891]
 [0.00698282 0.94709335 0.03919805 0.00672579]
 [0.00748544 0.03479346 0.9549025  0.00281859]
 [0.01202712 0.02667833 0.00885633 0.95243822]]
StarcoderdataJupyterStructuredCleanDedup