def build_lstm_vocab(tokens):
    """

    Builds vocab from list of sequences

    :param sentences:  list of sequences, each a list of tokens
    :return: tuple of (dictionary: word --> unique index, pad_token_idx)
    """
    all_tokens = list(set(tokens))

    vocab = {token: i for i, token in enumerate(all_tokens)}

    return vocab