def _ExtractTokenization(encoded_filepath, vocab):
  """Maps the words in the encoded file to their list of token ids.

  Reads all the subwords in encoded file. Concatenates them while they have @@
  as their last two characters. The last token of a word is the subword without
  @@. Maps the full word to the list of corresponding vocab ids of the subwords
  from the vocab dictionary.

  Args:
    encoded_filepath: String, filepath of the BPE encoded file.
    vocab: Dictionary of subwords (string) to token ids (int).

  Returns:
    Dictionary of words (string) to list of token ids (list of int).
  """
  word_tokenization = {}
  with open(encoded_filepath, 'r') as encoded_file:
    for line in encoded_file:
      full_word = ''
      ids = []
      for word in line.strip('\r\n ').split(' '):
        ids.append(vocab[word])
        if word[-2:] == '@@':
          full_word += word[:-2]
        else:
          full_word += word
          word_tokenization[full_word] = ids
          full_word = ''
          ids = []
  return word_tokenization