from keras.models import Sequential
from keras.layers.core import Dense, Activation, Dropout, Merge, TimeDistributedDense
from keras.layers.recurrent import LSTM
from keras.optimizers import RMSprop, SGD, Adagrad
from keras.regularizers import l2

import math

class TwoLayerLSTM:

  def __init__(self, hidden_size, vocab_size, dropin, droph, optimiser, l2reg):
    self.hidden_size = hidden_size # number of units in first LSTM
    self.dropin = dropin # prob. of dropping input units
    self.droph = droph # prob. of dropping hidden->hidden units
    self.vocab_size = vocab_size # size of word vocabulary
    self.optimiser = optimiser # optimisation method
    self.l2reg = l2reg # weight regularisation penalty

  def buildKerasModel(self):
    '''
    Define the exact structure of your model here. We create an image
    description generation model by merging the VGG image features with
    a word embedding model, with an LSTM over the sequences.

    The order in which these appear below (text, image) is _IMMUTABLE_.
    '''

    print('Building Keras model...')

    # We will learn word representations for each word
    text = Sequential()
    text.add(TimeDistributedDense(self.vocab_size, self.hidden_size, 
                                  W_regularizer=l2(self.l2reg)))
    text.add(Dropout(self.dropin))
    
    # Compress the 4096D VGG FC_7 features into hidden_size
    visual = Sequential()
    visual.add(TimeDistributedDense(4096, self.hidden_size, 
                                    W_regularizer=l2(self.l2reg)))
    text.add(Dropout(self.dropin))

    # The model is a merge of the VGG features and the Word Embedding vectors
    model = Sequential()
    model.add(Merge([text, visual], mode='sum'))
    model.add(LSTM(self.hidden_size, self.hidden_size, return_sequences=True)) # First LSTM layer 

    # The second layer has 80% of the units of the first layer
    stacked_LSTM_size = int(math.floor(self.hidden_size * 0.8))
    model.add(Dropout(self.droph))
    model.add(LSTM(self.hidden_size, stacked_LSTM_size, return_sequences=True)) # Second LSTM layer 
    model.add(TimeDistributedDense(stacked_LSTM_size, self.vocab_size, W_regularizer=l2(self.l2reg)))
    model.add(Activation('time_distributed_softmax'))
    
    model.compile(loss='categorical_crossentropy', optimizer=self.optimiser)

    return model


@@ -352,16 +352,32 @@
 r, l2reg
+, weights = None
 ):%0A    s
@@ -725,16 +725,83 @@
  penalty
+%0A    self.weights = weights # initialise with checkpointed weights?
 %0A%0A  def 
@@ -2412,16 +2412,149 @@
 miser)%0A%0A
+    if self.weights != None:%0A      # Initialise the weights of the model%0A      model.load_weights(%22%25s/weights.hdf5%22 %25 self.weights)%0A%0A
     retu

Allow a model to use checkpointed weights