'''
Created on Jan, 2017

@author: hugo

'''
from __future__ import absolute_import
import argparse
from os import path
import numpy as np

from autoencoder.core.vae import VarAutoEncoder, load_vae_model, save_vae_model
from autoencoder.preprocessing.preprocessing import load_corpus, doc2vec, vocab_weights
from autoencoder.utils.op_utils import vecnorm
from autoencoder.utils.io_utils import dump_json


def train(args):
    corpus = load_corpus(args.input)
    vocab, docs, word_freq = corpus['vocab'], corpus['docs'], corpus['word_freq']
    n_vocab = len(vocab)
    n_docs = len(docs)

    X_docs = [vecnorm(doc2vec(x, n_vocab), 'logmax1', 0) for x in docs.values()]

    # Prepare feature_weights for weighted loss
    feature_weights = None

    np.random.seed(0)
    np.random.shuffle(X_docs)
    n_val = args.n_val
    X_train = np.r_[X_docs[:-n_val]]
    X_val = np.r_[X_docs[-n_val:]]

    vae = VarAutoEncoder(n_vocab, args.n_intermediate_dim, args.n_dim, args.batch_size, weights_file=args.load_weights)
    vae.fit([X_train[:3000], X_train[:3000]], [X_val, X_val], nb_epoch=args.n_epoch)

    if args.save_model:
        arch_file  = args.save_model + '.arch'
        weights_file  = args.save_model + '.weights'
        save_vae_model(vae, arch_file, weights_file)
    print 'Saved model arch and weights file to %s and %s, respectively.' \
            % (arch_file, weights_file)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input corpus file')
    parser.add_argument('-nid', '--n_intermediate_dim', type=int, default=512, help='num of intermediate dimensions (default 512)')
    parser.add_argument('-nd', '--n_dim', type=int, default=128, help='num of dimensions (default 128)')
    parser.add_argument('-ne', '--n_epoch', type=int, default=100, help='num of epoches (default 100)')
    parser.add_argument('-bs', '--batch_size', type=int, default=100, help='batch size (default 100)')
    parser.add_argument('-nv', '--n_val', type=int, default=1000, help='size of validation set (default 1000)')
    parser.add_argument('-lw', '--load_weights', type=str, help='path to the pretrained weights file')
    parser.add_argument('-sm', '--save_model', type=str, default='model', help='path to the output model')
    args = parser.parse_args()

    train(args)

if __name__ == '__main__':
    main()

@@ -77,16 +77,30 @@
 _import%0A
+import timeit%0A
 import a
@@ -471,16 +471,18 @@
 ut)%0A    
+n_
 vocab, d
@@ -488,22 +488,15 @@
 docs
-, word_freq = 
+ = len(
 corp
@@ -506,16 +506,17 @@
 'vocab'%5D
+)
 , corpus
@@ -527,79 +527,89 @@
 cs'%5D
-,
+%0A   
  corpus
-%5B'word_freq'%5D%0A    n_vocab = len(vocab)%0A    n_docs = len(docs)%0A%0A
+.clear() # save memory%0A%0A    X_docs = %5B%5D%0A    for k in docs.keys():%0A    
     
@@ -614,20 +614,24 @@
   X_docs
- = %5B
+.append(
 vecnorm(
@@ -638,17 +638,23 @@
 doc2vec(
-x
+docs%5Bk%5D
 , n_voca
@@ -674,161 +674,141 @@
 , 0)
- for x in docs.values()%5D%0A%0A    # Prepare feature_weights for weighted loss%0A    feature_weights = None%0A%0A    np.random.seed(0)%0A    np.random.shuffle(X_docs)
+)%0A        del docs%5Bk%5D%0A%0A    np.random.seed(0)%0A    np.random.shuffle(X_docs)%0A    # X_docs_noisy = corrupted_matrix(np.r_%5BX_docs%5D, 0.1)%0A
 %0A   
@@ -830,16 +830,18 @@
 _val%0A   
+ #
  X_train
@@ -869,16 +869,18 @@
 al%5D%5D%0A   
+ #
  X_val =
@@ -903,16 +903,219 @@
 _val:%5D%5D%0A
+    X_train = np.r_%5BX_docs%5B:-n_val%5D%5D%0A    del X_docs%5B:-n_val%5D%0A    X_val = np.r_%5BX_docs%5D%0A    del X_docs%0A%0A    start = timeit.default_timer()%0A%0A    size = X_train.shape%5B0%5D - X_train.shape%5B0%5D %25 args.batch_size
 %0A    vae
@@ -1249,20 +1249,20 @@
 _train%5B:
-3000
+size
 %5D, X_tra
@@ -1269,12 +1269,12 @@
 in%5B:
-3000
+size
 %5D%5D, 
@@ -1313,16 +1313,77 @@
 epoch)%0A%0A
+    print 'runtime: %25ss' %25 (timeit.default_timer() - start)%0A%0A
     if a

update train_vae