<jupyter_start><jupyter_text>Preprocessing the input data<jupyter_code># concat last name and first name
sdf['name_last_name_first'] = sdf['name_last'] + ' ' + sdf['name_first']

# build n-gram list
vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=3, ngram_range=(NGRAMS, NGRAMS), lowercase=False) 
a = vect.fit_transform(sdf.name_last_name_first)
vocab = vect.vocabulary_

# sort n-gram by freq (highest -> lowest)
words = []
for b in vocab:
    c = vocab[b]
    #print(b, c, a[:, c].sum())
    words.append((a[:, c].sum(), b))
    #break
words = sorted(words, reverse=True)
words_list = [w[1] for w in words]
num_words = len(words_list)
print("num_words = %d" % num_words)


def find_ngrams(text, n):
    a = zip(*[text[i:] for i in range(n)])
    wi = []
    for i in a:
        w = ''.join(i)
        try:
            idx = words_list.index(w)
        except:
            idx = 0
        wi.append(idx)
    return wi

# build X from index of n-gram sequence
X = np.array(sdf.name_last_name_first.apply(lambda c: find_ngrams(c, NGRAMS)))

# check max/avg feature
X_len = []
for x in X:
    X_len.append(len(x))

max_feature_len = max(X_len)
avg_feature_len = int(np.mean(X_len))

print("Max feature len = %d, Avg. feature len = %d" % (max_feature_len, avg_feature_len))
y = np.array(sdf.race.astype('category').cat.codes)

# Split train and test dataset
X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)<jupyter_output>num_words = 2260<jupyter_text>Train a LSTM modelref: http://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/<jupyter_code>'''The dataset is actually too small for LSTM to be of any advantage
compared to simpler, much faster methods such as TF-IDF + LogReg.
Notes:

- RNNs are tricky. Choice of batch size is important,
choice of loss and optimizer is critical, etc.
Some configurations won't converge.

- LSTM loss decrease patterns during training can be quite different
from what you see with CNNs/MLPs/etc.
'''
import keras
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Embedding, Dropout, Activation
from keras.layers import LSTM
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.models import load_model

max_features = num_words # 20000
feature_len = 25 # avg_feature_len # cut texts after this number of words (among top max_features most common words)
batch_size = 32

print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')

print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=feature_len)
X_test = sequence.pad_sequences(X_test, maxlen=feature_len)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

num_classes = np.max(y_train) + 1
print(num_classes, 'classes')

print('Convert class vector to binary class matrix '
      '(for use with categorical_crossentropy)')
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
print('y_train shape:', y_train.shape)
print('y_test shape:', y_test.shape)
print('Build model...')

model = Sequential()
model.add(Embedding(num_words, 32, input_length=feature_len))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(num_classes, activation='softmax'))

# try using different optimizers and different optimizer configs
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print(model.summary())
print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, epochs=EPOCHS,
          validation_split=0.1, verbose=2)
score, acc = model.evaluate(X_test, y_test,
                            batch_size=batch_size, verbose=2)
print('Test score:', score)
print('Test accuracy:', acc)<jupyter_output>Train...
Train on 96387 samples, validate on 10710 samples
Epoch 1/15
 - 247s - loss: 1.4697 - acc: 0.5515 - val_loss: 1.2159 - val_acc: 0.6375
Epoch 2/15
 - 244s - loss: 1.1779 - acc: 0.6516 - val_loss: 1.1191 - val_acc: 0.6701
Epoch 3/15
 - 261s - loss: 1.1057 - acc: 0.6771 - val_loss: 1.0746 - val_acc: 0.6861
Epoch 4/15
 - 291s - loss: 1.0620 - acc: 0.6913 - val_loss: 1.0396 - val_acc: 0.6995
Epoch 5/15
 - 301s - loss: 1.0269 - acc: 0.7009 - val_loss: 1.0196 - val_acc: 0.7020
Epoch 6/15
 - 299s - loss: 0.9992 - acc: 0.7105 - val_loss: 1.0106 - val_acc: 0.7105
Epoch 7/15
 - 319s - loss: 0.9761 - acc: 0.7178 - val_loss: 1.0002 - val_acc: 0.7129
Epoch 8/15
 - 333s - loss: 0.9577 - acc: 0.7225 - val_loss: 0.9924 - val_acc: 0.7146
Epoch 9/15
 - 335s - loss: 0.9411 - acc: 0.7264 - val_loss: 0.9885 - val_acc: 0.7191
Epoch 10/15
 - 334s - loss: 0.9276 - acc: 0.7312 - val_loss: 0.9819 - val_acc: 0.7202
Epoch 11/15
 - 326s - loss: 0.9132 - acc: 0.7361 - val_loss: 0.9782 - val_acc: 0.7218
Epoc[...]<jupyter_text>Confusion Matrix<jupyter_code>y_pred = model.predict_classes(X_test, verbose=2)
p = model.predict_proba(X_test, verbose=2) # to predict probability
target_names = list(sdf.race.astype('category').cat.categories)
print(classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names))
print(confusion_matrix(np.argmax(y_test, axis=1), y_pred))<jupyter_output>precision    recall  f1-score   support

     Asian,GreaterEastAsian,EastAsian       0.86      0.79      0.82      1099
      Asian,GreaterEastAsian,Japanese       0.91      0.89      0.90      1467
             Asian,IndianSubContinent       0.78      0.76      0.77      1572
              GreaterAfrican,Africans       0.56      0.44      0.49       734
                GreaterAfrican,Muslim       0.67      0.66      0.67      1248
              GreaterEuropean,British       0.76      0.89      0.82      8289
         GreaterEuropean,EastEuropean       0.76      0.76      0.76      1666
               GreaterEuropean,Jewish       0.51      0.43      0.47      2048
  GreaterEuropean,WestEuropean,French       0.73      0.58      0.64      2459
GreaterEuropean,WestEuropean,Germanic       0.51      0.41      0.46       774
GreaterEuropean,WestEuropean,Hispanic       0.71      0.69      0.70      2082
 GreaterEuropean,WestEuropean,Italian       0.74   [...]<jupyter_text>Save model<jupyter_code>model.save('./models/wiki/lstm/wiki_name_lstm.h5')
words_df = pd.DataFrame(words_list, columns=['vocab'])
words_df.to_csv('./models/wiki/lstm/wiki_name_vocab.csv', index=False, encoding='utf-8')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup