#!/usr/bin/env python

import logging
import optparse
import subprocess
import sys
import os

def main():
  logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
  parser = optparse.OptionParser("%prog [options]")
  parser.add_option("-w", "--working-dir", dest="working_dir")
  parser.add_option("-c", "--corpus", dest="corpus_stem")
  parser.add_option("-l", "--nplm-home", dest="nplm_home")
  parser.add_option("-e", "--epochs", dest="epochs", type="int")
  parser.add_option("-n", "--ngram-size", dest="ngram_size", type="int")
  parser.add_option("-b", "--minibatch-size", dest="minibatch_size", type="int")
  parser.add_option("-s", "--noise", dest="noise", type="int")
  parser.add_option("-d", "--hidden", dest="hidden", type="int")
  parser.add_option("-i", "--input-embedding", dest="input_embedding", type="int")
  parser.add_option("-o", "--output-embedding", dest="output_embedding", type="int")
  parser.add_option("-t", "--threads", dest="threads", type="int")
  parser.add_option("-m", "--output-model", dest="output_model")
  parser.add_option("-o", "--output-dir", dest="output_dir")
  parser.add_option("-p", "--skip-preparation", action="store_true", dest="skip_preparation")
  parser.add_option("-f", "--config-options-file", dest="config_options_file")
  parser.add_option("-g", "--log-file", dest="log_file")

  parser.set_defaults(
    working_dir = "working"
    ,corpus_stem = "train.10k"
    ,nplm_home = "/home/bhaddow/tools/nplm"
    ,epochs = 10
    ,ngram_size = 14
    ,minibatch_size=1000
    ,noise=100
    ,hidden=750
    ,input_embedding=150
    ,output_embedding=150
    ,threads=1
    ,output_model = "train.10k"
    ,output_dir = None
    ,skip_preparation = False
    ,config_options_file = "config"
    ,log_file = "log"
  )

  options,args = parser.parse_args(sys.argv)

  # In order to allow for different models to be trained after the same
  # preparation step, we should provide an option for multiple output directories
  # If we have not set output_dir, set it to the same thing as the working dir

  if options.output_dir is None:
    options.output_dir = options.working_dir
  else:
    # Create output dir if necessary
    if not os.path.exists(options.output_dir):
      os.makedirs(options.output_dir)

  config_file = options.output_dir + "/" + options.config_options_file + '-' + options.output_model
  log_file = options.output_dir + "/" + options.log + '-' + options.output_model
  log_file_write = open(log_file, 'w')
  config_file_write = open(config_file, 'w')

  config_file_write.write("Called: " + ' '.join(sys.argv) + '\n\n')


  in_file = options.working_dir + "/" + options.corpus_stem + ".ngrams"
  vocab_file = options.working_dir + "/vocab"
  prep_file = options.working_dir + "/" + options.output_model + ".prepared"

  if (not options.skip_preparation):
    prep_args = [options.nplm_home + "/src/prepareNeuralLM", "--train_text", in_file, "--ngram_size", \
                  str(options.ngram_size), "--ngramize", "0", "--words_file", vocab_file, "--train_file", prep_file ]
    print "Prepare model command: "
    print ', '.join(prep_args)
    config_file_write.write("Preparation step:\n" + ' '.join(prep_args) + '\n\n')

    log_file_write.write("Preparation output:\n")
    ret = subprocess.call(prep_args, stdout=log_file_write, stderr=log_file_write)
    log_file_write.write("\n")
    if ret: raise Exception("Prepare failed")

  model_prefix = options.output_dir + "/" + options.output_model + ".model.nplm"
  train_args = [options.nplm_home + "/src/trainNeuralNetwork", "--train_file", prep_file, "--num_epochs", str(options.epochs),
                "--input_words_file", vocab_file, "--output_words_file", vocab_file, "--model_prefix",
                model_prefix, "--learning_rate", "1", "--minibatch_size", str(options.minibatch_size),
                "--num_noise_samples", str(options.noise), "--num_hidden", str(options.hidden), "--input_embedding_dimension",
                str(options.input_embedding), "--output_embedding_dimension", str(options.output_embedding), "--num_threads",
                str(options.threads)]
  print "Train model command: "
  print ', '.join(train_args)

  config_file_write.write("Training step:\n" + ' '.join(train_args) + '\n')
  config_file_write.close()

  log_file_write.write("Training output:\n")
  ret = subprocess.call(train_args, stdout=log_file_write, stderr=log_file_write)
  if ret: raise Exception("Training failed")

if __name__ == "__main__":
  main()




#EPOCHS=10
#NGRAM_SIZE=14
#MINIBATCH_SIZE=1000
#NOISE=100
#HIDDEN=750
#INPUT_EMBEDDING=150
#OUTPUT_EMBEDDING=150
#THREADS=8
#

#$ROOT/src/prepareNeuralLM --train_text $INFILE --ngram_size $NGRAM_SIZE --ngramize 0  --words_file $VOCAB --train_file $WORKDIR/train.ngrams || exit 1

#$ROOT/src/trainNeuralNetwork --train_file $WORKDIR/train.ngrams  \
#   --num_epochs $EPOCHS --input_words_file $VOCAB --output_words_file $VOCAB --model_prefix $WORKDIR/$PREFIX \
#   --learning_rate 1 --minibatch_size $MINIBATCH_SIZE --num_noise_samples $NOISE --num_hidden $HIDDEN \
#   --input_embedding_dimension $INPUT_EMBEDDING --output_embedding_dimension $OUTPUT_EMBEDDING --num_threads $THREADS || exit 1



@@ -1113,33 +1113,33 @@
 er.add_option(%22-
-o
+r
 %22, %22--output-dir
@@ -2465,16 +2465,21 @@
 ions.log
+_file
  + '-' +

Fix small oversights