<jupyter_start><jupyter_text>Read the data from the file and remove all 14 punctuations<jupyter_code>data = open('pg11.txt').read().lower()

punc = """!()-[]{};:'"\,<>/?@#$%^&*_~\ufeff"""
punc = punc
new = ''
for i in data:
    if i not in punc:
        new = new + i
new = new.replace('.', ' ')        
chars = sorted(list(set(new)))
char_to_num = dict((x, i) for i,x in enumerate(chars))
print(char_to_num)<jupyter_output>{'\n': 0, ' ': 1, '0': 2, '3': 3, 'a': 4, 'b': 5, 'c': 6, 'd': 7, 'e': 8, 'f': 9, 'g': 10, 'h': 11, 'i': 12, 'j': 13, 'k': 14, 'l': 15, 'm': 16, 'n': 17, 'o': 18, 'p': 19, 'q': 20, 'r': 21, 's': 22, 't': 23, 'u': 24, 'v': 25, 'w': 26, 'x': 27, 'y': 28, 'z': 29}<jupyter_text>Split by newline character and removed all empy lines<jupyter_code>new_data = new.split('\n')
count = 0

while count < len(new_data):
    if new_data[count] == '':
        new_data.pop(count)
    else:
        count += 1
        
print(len(new_data))
print(new_data[:5])
# Average line length

summed = 0
for i in new_data:
    summed = summed + len(i)
    
print(summed / len(new_data) )
from google.colab import drive
drive.mount('/content/drive')
from tensorflow.keras.layers import Dense, Dropout, LSTM
from tensorflow.keras.models import Sequential
from tensorflow.keras import utils
filepath = '/content/drive/My Drive/EIP/a6.hdf5'
ckpt = tf.keras.callbacks.ModelCheckpoint(filepath, verbose=1, monitor='loss', save_best_only=True, mode='min')
callbacks = [ckpt]
from tensorflow.keras.preprocessing.sequence import pad_sequences
X, y = [], []
i = 0
for line in new_data:
    while len(line) > 50:   
        X.insert(i, [char_to_num[j] for j in list(line[:50])])    
        y.insert(i, char_to_num[line[50]])
        i+=1
        line = line[51:]
        
    if len(line) > 1:
        X.insert(i, [char_to_num[j] for j in list(line[:-1])])    
        y.insert(i, char_to_num[line[-1]])
        
X = pad_sequences(X)
X = utils.to_categorical(X)
X = X / len(chars)
y = utils.to_categorical(y)
print(X.shape)
print(y.shape)
model = Sequential([
    Dropout(0.1),
    LSTM(256, input_shape=(X.shape[1], X.shape[2])),
    Dense(30, activation='softmax')
])

model.compile(loss='categorical_crossentropy', optimizer='adam')
model.fit(X, y, epochs=100, batch_size=32)
model.load_weights(filepath)
model.compile(loss='categorical_crossentropy', optimizer='adam')
import sys
int_to_chars = dict((i,x) for i,x in enumerate(chars))

pattern =  """Alice was beginning to get very tired of sitting""".lower()

print(f'Seed: {pattern}')
print(f'\nPrediction:')

pattern = [char_to_num[j] for j in pattern]

for i in range(500):
    x = utils.to_categorical(pattern, num_classes=30)
    x = np.expand_dims(x, axis=0)
    pred = np.argmax(np.squeeze(model.predict(x)))
    pattern.pop(0)
    pattern.append(pred)
    print(int_to_chars[pred], end='')<jupyter_output>Seed: alice was beginning to get very tired of sitting

Prediction:
 band jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jound jo
StarcoderdataJupyterStructuredCleanDedup