<jupyter_start><jupyter_text>Based on Hugging Face Transformershttps://github.com/huggingface/transformers<jupyter_code>#!pip install transformers
#!pip install keras
#!pip install tensorflow==1.15<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>import transformers as ppb
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from keras.utils.np_utils import to_categorical
import torch
import math
# For DistilBERT:
#model_class, tokenizer_class, pretrained_weights = (ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-cased')

## Want BERT instead of distilBERT? Uncomment the following line:
model_class, tokenizer_class, pretrained_weights = (ppb.BertModel, ppb.BertTokenizer, 'bert-base-cased')

# Load pretrained model/tokenizer
tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
model = model_class.from_pretrained(pretrained_weights)
df = pd.read_csv("../datasets/NER/train.tsv", sep="\t", header=None)
df = df.replace(np.nan, '', regex=True)
test_df = pd.read_csv("../datasets/NER/test.tsv", sep="\t", header=None)
test_df = test_df.replace(np.nan, '', regex=True)
# Truncate long sentences to 128 tokens
X = df[0].apply((lambda x: tokenizer.encode(x, add_special_tokens=True, max_length=128)))
y = np.array(df[1])
y = ['missing' if x is np.nan else x for x in y]
del df

X_test = test_df[0].apply((lambda x: tokenizer.encode(x, add_special_tokens=True, max_length=128)))
y_test = np.array(test_df[1])
y_test = ['missing' if x is np.nan else x for x in y_test]
del test_df
# One hot Encoding of y
encoder = LabelEncoder()
encoder.fit(y)

y = encoder.transform(y)
y = to_categorical(y)

# One hot Encoding of y test
y_oh = encoder.transform(y_test)
y_oh = to_categorical(y_oh)
def GetEmbeddings(tokenizedBatch):
    max_len = 0
    for i in tokenizedBatch.values:
        if len(i) > max_len:
            max_len = len(i)

    padded = np.array([i + [0]*(max_len-len(i)) for i in tokenizedBatch.values])
    
    attention_mask = np.where(padded != 0, 1, 0)
    
    input_ids = torch.tensor(padded).to(torch.long)  
    attention_mask = torch.tensor(attention_mask)

    with torch.no_grad():
        last_hidden_states = model(input_ids, attention_mask=attention_mask)
    
    features = last_hidden_states[0][:,0,:].numpy()
    return features
BATCH_SIZE = 5000
all_embeddings = []
all_embeddings_test = []

# Process Training Set Embeddings
batches = math.ceil(X.shape[0] / BATCH_SIZE)

for i in range(1, batches+1):
    print("Generating Embeddings for Batch:",i,"of", batches)
    batchEmbeddings = GetEmbeddings(X[(i-1)*BATCH_SIZE:i*BATCH_SIZE])
    all_embeddings.append(batchEmbeddings)

# Process Test Set Embeddings
batches = math.ceil(X_test.shape[0] / BATCH_SIZE)

for i in range(1, batches+1):
    print("Generating Test Embeddings for Batch:",i,"of", batches)
    batchEmbeddings = GetEmbeddings(X_test[(i-1)*BATCH_SIZE:i*BATCH_SIZE])
    all_embeddings_test.append(batchEmbeddings)
all_embeddings = np.concatenate(all_embeddings, axis=0)
all_embeddings_test = np.concatenate(all_embeddings_test, axis=0)
np.save('../binary/bert_NER_embeddings_twitter.npy', all_embeddings)
np.save('../binary/y_NER_twitter.npy', y)
np.save('../binary/bert_NER_embeddings_test_twitter.npy', all_embeddings_test)
np.save('../binary/y_test_NER_twitter.npy', y_oh)<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code>#import tensorflow.compat.v1 as tf
#tf.disable_v2_behavior()

import tensorflow as tf
import tensorflow.keras as keras 
from keras.layers import Input, Lambda, Dense
from keras.models import Model
from keras.callbacks import EarlyStopping
import keras.backend as K
from keras.optimizers import adam, sgd
all_embeddings = np.load('../binary/bert_NER_embeddings_twitter.npy')
y = np.load('../binary/y_NER_twitter.npy')
all_embeddings_test = np.load('../binary/bert_NER_embeddings_test_twitter.npy')
y_oh = np.load('../binary/y_test_NER_twitter.npy')
#sgd = sgd(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
optim = adam(lr=0.0003, beta_1=0.9, beta_2=0.999, amsgrad=False)
def build_model(): 
    embedding = Input(shape=(768,), dtype="float")
    dense1 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(embedding)
    dense2 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense1)
    dense3 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense2)
    dense4 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense3)
    dense5 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense4)
    dense6 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense5)
    dense7 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense6)
    dense8 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense7)
    dense9 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense8)
    dense10 = Dense(1000, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(dense9)
    pred = Dense(4, activation='sigmoid')(dense9)
    model = Model(inputs=[embedding], outputs=pred)
    model.compile(loss='categorical_crossentropy', optimizer=optim, metrics=['accuracy'], )
    return model
model_bert = build_model()
model_bert.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, 768)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 1000)              769000    
_________________________________________________________________
dense_2 (Dense)              (None, 1000)              1001000   
_________________________________________________________________
dense_3 (Dense)              (None, 1000)              1001000   
_________________________________________________________________
dense_4 (Dense)              (None, 1000)              1001000   
_________________________________________________________________
dense_5 (Dense)              (None, 1000)              1001000   
_________________________________________________________________
dense_6 (D[...]<jupyter_text>Change to training set<jupyter_code>es = EarlyStopping(monitor='loss', patience=15)
cb_list = [es]
with tf.Session() as session:
    K.set_session(session)
    #tf.compat.v1.keras.backend.set_session(session)
    session.run(tf.global_variables_initializer())  
    session.run(tf.tables_initializer())
    history = model_bert.fit(all_embeddings, y, epochs=1000, batch_size=10000, 
                             validation_split = 0.001, callbacks=cb_list)
    model_bert.save_weights('../model/bert_ner_logistic_twitter/model_bert_weights.h5')
import matplotlib.pyplot as plt
%matplotlib inline

#acc = history.history['acc']
acc = history.history['acc']
loss = history.history['loss']

epochs = range(1, len(acc) + 1)

plt.plot(epochs, acc, 'g', label='Training Acc')
plt.title('Training and validation Acc')
plt.xlabel('Epochs')
plt.ylabel('Acc')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Predict on Test Data<jupyter_code>bs = 10000
batches = math.ceil(all_embeddings_test.shape[0] / bs)
all_preds = []
all_probs = []
    
with tf.Session() as session:
    K.set_session(session)
    session.run(tf.global_variables_initializer())  
    session.run(tf.tables_initializer())
    model_bert.load_weights('../model/bert_ner_logistic_twitter/model_bert_weights.h5')

    for i in range(1,batches+1):
        print("Predicting Batch",i)
        new_text_pr = all_embeddings_test[(i-1)*bs:i*bs]
        preds = model_bert.predict(new_text_pr)
        all_probs.append(preds)
        preds = encoder.inverse_transform(np.argmax(preds,axis=1))
        all_preds.append(preds)
results = np.concatenate(all_preds, axis=0)
results_probs = np.concatenate(all_probs, axis=0)
np.savetxt("../output/NER/bert_logistic_twitter/test_results.tsv", results_probs, delimiter="\t")
np.savetxt("../output/NER/bert_logistic_twitter/test_predictions.tsv", results, delimiter="\t", fmt='%s')
print("Accuracy: ",sum(results==y_test)/results.shape[0])
from sklearn.metrics import f1_score as f1_score
f1_score(y_test, results, average='macro')<jupyter_output><empty_output><jupyter_text>Most Common Class<jupyter_code>most_common_results = np.full((5381,), 'O')
most_common_results.shape
f1_score(y_test, most_common_results, average='macro')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup