<jupyter_start><jupyter_text>Text Preprocessing Exploration<jupyter_code>import contractions
import re
print(contractions.fix("i've"))
print(contractions.fix("he's"))
# re.sub(r'[^\w\d\s\']+', '', 'I have $500 and I love that! I said "WHAT" eğitmeye')
import re
re.sub(r'[^a-zA-Z\d\s\']+', '', 'I have $500 and I love that! I said "WHAT" eğitmeye')
len('z'), 'Z'.lower()
# TF IDF Exploration
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
transformer = TfidfTransformer(smooth_idf=False)
count_vectorizer = CountVectorizer(ngram_range=(1, 2))
sentence = ["an apple a day keeps the doctor away"]
v = CountVectorizer(ngram_range=(1,2))
print(v.fit(sentence))
v.get_feature_names()
v.transform(sentence)
v.vocabulary_
transformer.fit(v.transform(sentence))
from numpy import array
from keras.preprocessing.text import one_hot 
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers.embeddings import Embedding
# define documents
docs = ['Well done!',
'Good work',
'Great effort',
'nice work',
'Excellent!',
'Weak',
'Poor effort!',
'not good',
'poor work',
'Could have done better.']
# define class labels
labels = array([1,1,1,1,1,0,0,0,0,0])
# integer encode the documents
vocab_size = 50
encoded_docs = [one_hot(d, vocab_size) for d in docs]
print(encoded_docs)
[e for each in encoded_docs for e in each]
set([e for each in encoded_docs for e in each])
vocab_size = 50
for d in docs:
    print(d)
    print(one_hot(d, vocab_size))
one_hot("Well, poor effort", vocab_size)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup