<jupyter_start><jupyter_text>**1. Set up LSTM class and load model**<jupyter_code>char_to_int = {"'": 1, ',': 2, 'e': 3, 'a': 4, 'r': 5, 'i': 6, 's': 7, 'n': 8, 'o': 9, 't': 10, 'l': 11, 'c': 12, 'd': 13, 'm': 14, 'u': 15, 'h': 16, 'g': 17, 'p': 18, 'b': 19, 'k': 20, 'y': 21, '"': 22, 'f': 23, 'w': 24, 'v': 25, 'z': 26, 'j': 27, 'x': 28, 'q': 29, '-': 30, '.': 31, '[': 32, '1': 33, ']': 34}
int_to_char = {v: k for k, v in char_to_int.items()}
class LSTM(nn.Module):


    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers):
        """
        Initialize the model by setting up the layers.
        """
        super(LSTM, self).__init__()

        self.output_size = output_size
        self.n_layers = n_layers
        self.hidden_dim = hidden_dim
        
        # embedding and LSTM layers
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, batch_first=True)
        #dropout layers
        self.dropout = nn.Dropout(0.2)
        # linear layers
        self.fc1 = nn.Linear(hidden_dim, 256)
        self.fc2 = nn.Linear(256, output_size)

        

    def forward(self, x, hidden):
        """
        Perform a forward pass of our model on some input and hidden state.
        """
        batch_size = x.size(0)

        # embeddings and lstm_out
        x = x.long()
        embeds = self.embedding(x)
        lstm_out, hidden = self.lstm(embeds, hidden)
    
        # stack up lstm outputs
        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)
        out = self.dropout(lstm_out)
        # fully-connected layer
        out = nn.functional.relu(self.fc1(lstm_out))
        out = self.dropout(out)
        out = self.fc2(out)
        
        # reshape to be batch_size first
        out = out.view(batch_size, -1)
        out = out[:, -1] # get last batch of labels
        
        return out, hidden
    
    
    def init_hidden(self, batch_size):
        ''' Initializes hidden state '''
        # Create two new tensors with sizes n_layers x batch_size x hidden_dim,
        # initialized to zero, for hidden state and cell state of LSTM
        weight = next(self.parameters()).data
        
        if (train_on_gpu):
            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),
                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())
        else:
            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),
                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())
        
        return hidden
def load_model(filename):
    save_filename = os.path.splitext(os.path.basename(filename))[0] + '.pt'
    return torch.load(save_filename)
    
net_lstm = load_model('./save/trained_lstm')<jupyter_output><empty_output><jupyter_text>**2. functions for deploying model**<jupyter_code>def split_sentences(text):
    text = text.replace('\n', ' ')
    sentences = re.split(r'[.!?]+', text)
    print(sentences)
    return sentences

def split_words(sent):
    sent = re.sub(r'[^\w\s]','',sent)       #clean the data
    words = sent.split(' ')
    while("" in words) : 
        words.remove("") 
    return words

def tokenize_word(words):
    test_ints = []
    for word in words:
      word = word.lower()
      test_int = [char_to_int[char] for char in word if char in char_to_int]
      if len(test_int) > 0:
          test_ints.append(test_int)
    test_ints = pad_words(test_ints, 28)    #28 is the max length of dictionary used in training
    return test_ints
def pad_words(words, length):
    features = np.zeros((len(words), length), dtype=int)
    for i, row in enumerate(words):
        features[i, -len(row):] = np.array(row)[:length]
    return features
def predict(net, words):
    
    net.eval()

    test_ints = tokenize_word(words)    #convert words into ints

    features = np.array(test_ints)

    feature_tensor = torch.from_numpy(features)
    batch_size =  feature_tensor.size(0)
    h = net.init_hidden(batch_size)

    if(train_on_gpu):
        feature_tensor = feature_tensor.cuda()
    
    # get the output from the model
    output, h = net(feature_tensor, h)
    
    pred = torch.round(output.squeeze()/1000)

    return torch.sum(pred)

def txt_predict(net, filename):
    with open(filename, 'r') as f:
        data = f.read()
    
    # Split data into sentences and compute number of sentences in the text
    sentences = split_sentences(data) # a list of sent
    num_sents = len(sentences)

    num_words = 0
    num_syllables = 0
    for sent in sentences:
        words_to_predict = split_words(sent)
        num_words += len(words_to_predict)
        if len(words_to_predict) > 0:
            num_syllables += predict(net, words_to_predict)
    
    reading_level = 0.39 * (num_words/num_sents) + 11.8 * (num_syllables/num_words) - 15.59
    return reading_level

# txt_predict(net_lstm, '/content/mobydick.txt')
txt_predict(net_lstm, '/content/moby dick chap 64.txt')<jupyter_output>['CHAPTER 64', " <NAME>upper", "   Stubb's whale had been killed some distance from the ship", ' It was a calm; so, forming a tandem of three boats, we commenced the slow business of towing the trophy to the Pequod', ' And now, as we eighteen men with our thirty-six arms, and one hundred and eighty thumbs and fingers, slowly toiled hour after hour upon that inert, sluggish corpse in the sea; and it seemed hardly to budge at all, except at long intervals; good evidence was hereby furnished of the enormousness of the mass we moved', ' For, upon the great canal of Hang-Ho, or whatever they call it, in China, four or five laborers on the foot-path will draw a bulky freighted junk at the rate of a mile an hour; but this grand argosy we towed heavily forged along, as if laden with pig-lead in bulk', "  Darkness came on; but three lights up and down in the Pequod's main-rigging dimly guided our way; till drawing nearer we saw Ahab dropping one of several more lanterns over the bulwarks", '[...]
StarcoderdataJupyterStructuredCleanDedup