<jupyter_start><jupyter_text>import sys!{sys.executable} -m pip install tensorflow-hub<jupyter_code>## Get & Process Data<jupyter_output><empty_output><jupyter_text>conn = sqlite3.connect("db.sqlite3")df = pd.read_sql_query("select * from submissions;", conn)df.shape df = pd.·%&·$%&·$%&read_sql_query('db.sqlite3')df.head(20) df['clean_text'] = df['text'].str.replace('[^\w\s]',' ')df['clean_title'] = df['title'].str.replace('[^\w\s]',' ')df['combo'] = df['clean_text'] + df['clean_title']df.head()<jupyter_code>## Preprocessing<jupyter_output><empty_output><jupyter_text>df[['subreddit', 'combo']].head(1)<jupyter_code>nlp = spacy.load("en_core_web_md")

tokenizer = Tokenizer(nlp.vocab)
tokens = []

""" Make them tokens """

#stop words
STOP_WORDS = nlp.Defaults.stop_words.union(['', ' ', '-', 'reddit', 'post'])
    
tokens = []

for doc in tokenizer.pipe(df['combo'], batch_size=500):
    
    doc_tokens = []
    
    for token in doc: 
        if ((token.text.lower() not in STOP_WORDS) and 
            (token.is_stop == False) and 
            (token.is_punct == False) and 
            (token.pos_ != 'PRON')):
                 doc_tokens.append(token.lemma_.lower())
    tokens.append(' '.join(doc_tokens))
    
df['tokens'] = tokens
tfidf_vectorizer = TfidfVectorizer(min_df=0.02, max_df=.98,
                                   ngram_range=(1,2))
X = tfidf_vectorizer.fit_transform(df['tokens'])
y = pd.get_dummies(df['subreddit']).values #target

print(X.shape)
print(y.shape)<jupyter_output>(50000, 215)
(50000, 100)<jupyter_text>TF Model<jupyter_code>model = keras.Sequential()
model.add(keras.layers.Dense(64, input_dim=X.shape[1], activation='relu'))
model.add(keras.layers.Dropout(0.2))
model.add(keras.layers.Dense(16, activation='relu'))
model.add(keras.layers.Dense(y.shape[1], activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
from sklearn.model_selection import train_test_split

X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2)
history = model.fit(X_train, y_train, epochs=10, batch_size=64)<jupyter_output>Epoch 1/10
40000/40000 [==============================] - 2s 60us/sample - loss: 4.3572 - acc: 0.0345
Epoch 2/10
40000/40000 [==============================] - 2s 54us/sample - loss: 4.0373 - acc: 0.0834
Epoch 3/10
40000/40000 [==============================] - 2s 55us/sample - loss: 3.9464 - acc: 0.1010
Epoch 4/10
40000/40000 [==============================] - 2s 55us/sample - loss: 3.8887 - acc: 0.1128
Epoch 5/10
40000/40000 [==============================] - 2s 54us/sample - loss: 3.8472 - acc: 0.1178
Epoch 6/10
40000/40000 [==============================] - 2s 57us/sample - loss: 3.8189 - acc: 0.1245
Epoch 7/10
40000/40000 [==============================] - 2s 54us/sample - loss: 3.7950 - acc: 0.1283
Epoch 8/10
40000/40000 [==============================] - 2s 58us/sample - loss: 3.7786 - acc: 0.1323
Epoch 9/10
40000/40000 [==============================] - 2s 59us/sample - loss: 3.7614 - acc: 0.1337
Epoch 10/10
40000/40000 [==============================] - 2s 55us/sample - loss: [...]<jupyter_text>Test<jupyter_code>(np.sum(np.argmax(model.predict(X_test), axis=1) == np.argmax(y_test, axis=1)))/ X.shape[0]
model.predict_classes(X_test[:1])
df.to_pickle("./dummy.pkl")<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup