from nlpia.loaders import get_data
from tqdm import tqdm


df = get_data('deu')
input_texts, target_texts = [], []  # <1>
input_vocabulary = set()  # <3>
output_vocabulary = set()
start_token = '\t'  # <4>
stop_token = '\n'
max_training_samples = min(25000, len(df) - 1)  # <6>

for input_text, target_text in tqdm(zip(df.eng, df.deu), total=len(df)):
    target_text = start_token + target_text \
        + stop_token  # <7>
    input_texts.append(input_text)
    target_texts.append(target_text)
    for char in input_text:  # <8>
        if char not in input_vocabulary:
            input_vocabulary.add(char)
    for char in target_text:
        if char not in output_vocabulary:
            output_vocabulary.add(char)

input_vocabulary = sorted(input_vocabulary)  # <1>
output_vocabulary = sorted(output_vocabulary)

input_vocab_size = len(input_vocabulary)  # <2>
output_vocab_size = len(output_vocabulary)
max_encoder_seq_length = max(
    [len(txt) for txt in input_texts])  # <3>
max_decoder_seq_length = max(
    [len(txt) for txt in target_texts])

input_token_index = dict([(char, i) for i, char in
                          enumerate(input_vocabulary)])  # <4>
target_token_index = dict(
    [(char, i) for i, char in enumerate(output_vocabulary)])
reverse_input_char_index = dict((i, char) for char, i in
                                input_token_index.items())  # <5>
reverse_target_char_index = dict((i, char) for char, i in
                                 target_token_index.items())

import numpy as np  # <1>  # noqa


encoder_input_data = np.zeros((len(input_texts),
                               max_encoder_seq_length, input_vocab_size),
                              dtype='float32')  # <2>
decoder_input_data = np.zeros((len(input_texts),
                               max_decoder_seq_length, output_vocab_size),
                              dtype='float32')
decoder_target_data = np.zeros((len(input_texts),   
                                max_decoder_seq_length, output_vocab_size),
                               dtype='float32')
for i, (input_text, target_text) in enumerate(tqdm(
        zip(input_texts, target_texts), total=len(target_texts))):  # <3>
    for t, char in enumerate(input_text):  # <4>
        encoder_input_data[
            i, t, input_token_index[char]] = 1.  # <5>
    for t, char in enumerate(target_text):  # <6>
        decoder_input_data[
            i, t, target_token_index[char]] = 1.
        if t > 0:
            decoder_target_data[i, t - 1, target_token_index[char]] = 1


from keras.models import Model  # noqa
from keras.layers import Input, LSTM, Dense  # noqa


batch_size = 64    # <1>
epochs = 100       # <2>
num_neurons = 256  # <3>

encoder_inputs = Input(shape=(None, input_vocab_size))
encoder = LSTM(num_neurons, return_state=True)
encoder_outputs, state_h, state_c = encoder(encoder_inputs)
encoder_states = [state_h, state_c]

decoder_inputs = Input(shape=(None, output_vocab_size))
decoder_lstm = LSTM(num_neurons, return_sequences=True,
                    return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
                                     initial_state=encoder_states)
decoder_dense = Dense(output_vocab_size, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
              metrics=['acc'])
model.fit([encoder_input_data, decoder_input_data],
          decoder_target_data, batch_size=batch_size, epochs=epochs,
          validation_split=0.1)  # <4>

@@ -188,94 +188,48 @@
 oken
- = '%5Ct'  # %3C4%3E%0Astop_token = '%5Cn'%0Amax_training_samples = min(25000, len(df) - 1)  # %3C6%3E
+, stop_token = '%5Ct%5Cn'  # %3C2%3E%0An = len(df)
 %0A%0Afo
@@ -289,23 +289,17 @@
 , total=
-len(df)
+n
 ):%0A    t
@@ -1482,17 +1482,16 @@
 # noqa%0A%0A
-%0A
 encoder_
@@ -1517,56 +1517,10 @@
 os((
-len(input_texts),%0A                              
+n,
  max
@@ -1648,56 +1648,10 @@
 os((
-len(input_texts),%0A                              
+n,
  max
@@ -1774,60 +1774,10 @@
 os((
-len(input_texts),   %0A                               
+n,
  max
@@ -2342,16 +2342,100 @@
 %5D = 1%0A%0A%0A
+import pandas as pd  # noqa%0Aencoder_input_data = pd.DataFrame(encoder_input_data)%0A%0A%0A
 from ker
@@ -3342,16 +3342,475 @@
 'acc'%5D)%0A
+%0A%0Aimport os  # noqa%0Afrom keras.callbacks import ModelCheckpoint  # noqa%0Afrom nlpia.constants import BIGDATA_PATH  # noqa%0Acheckpoint_path = os.path.join(BIGDATA_PATH, 'checkpoints')%0Acheckpoint_path = os.path.join(checkpoint_path, 'nlpia-seq2seq-translation-weights.%7Bepoch:02d%7D-%7Bval_loss:.2f%7D.hdf5')%0A%0A%0Acheckpoint_callback = ModelCheckpoint(checkpoint_path,%0A                                      monitor='val_loss', verbose=0, save_best_only=False, mode='auto')%0A
 model.fi
@@ -3883,16 +3883,69 @@
 et_data,
+%0A          callbacks=%5Bcheckpoint_callback%5D,%0A         
  batch_s
@@ -3974,26 +3974,16 @@
 =epochs,
-%0A         
  validat

save checkpoints