import os
import pickle
import copy
import numpy as np


CODES = {'<PAD>': 0, '<EOS>': 1, '<UNK>': 2, '<GO>': 3 }


def load_data(path):
    """
    Load Dataset from File
    """
    input_file = os.path.join(path)
    with open(input_file, 'r', encoding='utf-8') as f:
        data = f.read()

    return data


def preprocess_and_save_data(source_path, target_path, text_to_ids):
    """
    Preprocess Text Data.  Save to to file.
    """
    # Preprocess
    source_text = load_data(source_path)
    target_text = load_data(target_path)

    source_text = source_text.lower()
    target_text = target_text.lower()

    source_vocab_to_int, source_int_to_vocab = create_lookup_tables(source_text)
    target_vocab_to_int, target_int_to_vocab = create_lookup_tables(target_text)

    source_text, target_text = text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int)

    # Save Data
    pickle.dump((
        (source_text, target_text),
        (source_vocab_to_int, target_vocab_to_int),
        (source_int_to_vocab, target_int_to_vocab)), open('preprocess.p', 'wb'))


def load_preprocess():
    """
    Load the Preprocessed Training data and return them in batches of <batch_size> or less
    """
    return pickle.load(open('preprocess.p', mode='rb'))


def create_lookup_tables(text):
    """
    Create lookup tables for vocabulary
    """
    vocab = set(text.split())
    vocab_to_int = copy.copy(CODES)

    for v_i, v in enumerate(vocab, len(CODES)):
        vocab_to_int[v] = v_i

    int_to_vocab = {v_i: v for v, v_i in vocab_to_int.items()}

    return vocab_to_int, int_to_vocab


def save_params(params):
    """
    Save parameters to file
    """
    pickle.dump(params, open('params.p', 'wb'))


def load_params():
    """
    Load parameters from file
    """
    return pickle.load(open('params.p', mode='rb'))


def batch_data(source, target, batch_size):
    """
    Batch source and target together
    """
    for batch_i in range(0, len(source)//batch_size):
        start_i = batch_i * batch_size
        source_batch = source[start_i:start_i + batch_size]
        target_batch = target[start_i:start_i + batch_size]
        yield np.array(pad_sentence_batch(source_batch)), np.array(pad_sentence_batch(target_batch))


def pad_sentence_batch(sentence_batch):
    """
    Pad sentence with <PAD> id
    """
    max_sentence = max([len(sentence) for sentence in sentence_batch])
    return [sentence + [CODES['<PAD>']] * (max_sentence - len(sentence))
            for sentence in sentence_batch]

@@ -276,14 +276,14 @@
     
-data =
+return
  f.r
@@ -293,25 +293,8 @@
 ()%0A%0A
-    return data%0A%0A
 %0Adef
@@ -887,16 +887,69 @@
 ve Data%0A
+    with open('preprocess.p', 'wb') as out_file:%0A    
     pick
@@ -958,16 +958,20 @@
 .dump((%0A
+    
         
@@ -998,32 +998,36 @@
 _text),%0A        
+   
+ 
 (source_vocab_to
@@ -1050,24 +1050,28 @@
 ab_to_int),%0A
+    
         (sou
@@ -1116,33 +1116,15 @@
 ), o
-pen('preprocess.p', 'wb')
+ut_file
 )%0A%0A%0A
@@ -1257,35 +1257,21 @@
 %22%22%22%0A    
-return pickle.load(
+with 
 open('pr
@@ -1285,32 +1285,79 @@
 s.p', mode='rb')
+ as in_file:%0A        return pickle.load(in_file
 )%0A%0A%0Adef create_l
@@ -1759,27 +1759,12 @@
     
-pickle.dump(params,
+with
  ope
@@ -1782,16 +1782,66 @@
 ', 'wb')
+ as out_file:%0A        pickle.dump(params, out_file
 )%0A%0A%0Adef 
@@ -1905,35 +1905,21 @@
 %22%22%22%0A    
-return pickle.load(
+with 
 open('pa
@@ -1937,16 +1937,63 @@
 de='rb')
+ as in_file:%0A        return pickle.load(in_file
 )%0A%0A%0Adef 

Use `with open()` to avoid leaking file handles