code: 
  
  
  
  
  
  
  # Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  ####################################################################################################
  
  #
  # Note: If when running this conversion script you're getting an exception:
  #     ModuleNotFoundError: No module named 'megatron.model.enums'
  # you need to tell python where to find the clone of Megatron-LM, e.g.:
  #
  # cd /tmp
  # git clone https://github.com/NVIDIA/Megatron-LM
  # PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
  #
  # if you already have it cloned elsewhere, simply adjust the path to the existing path
  #
  # If the training was done using a Megatron-LM fork, e.g.,
  # https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
  # in your path, i.e., /path/to/Megatron-DeepSpeed/
  #
  
  import argparse
  import os
  import re
  import zipfile
  
  import torch
  
  from transformers import MegatronBertConfig
  
  
  
  
  
  def    _lowerCamelCase	(	__lowerCamelCase   ,       __lowerCamelCase   ,       __lowerCamelCase=0					)   -> str:
  
       '''simple docstring'''
       # Format the message.
       if name is None:
            UpperCAmelCase__			:     Dict										= None
       else:
            UpperCAmelCase__			:     int										= """.""" * max(0   ,       spaces - 2					) + """# {:""" + str(50 - spaces					) + """s}"""
            UpperCAmelCase__			:     Union[str, Any]										= fmt.format(__lowerCamelCase					)
  
       # Print and recurse (if needed).
       if isinstance(__lowerCamelCase   ,       __lowerCamelCase					):
            if msg is not None:
                 print(__lowerCamelCase					)
            for k in val.keys():
                 recursive_print(__lowerCamelCase   ,       val[k]   ,       spaces + 2					)
       elif isinstance(__lowerCamelCase   ,       torch.Tensor					):
            print(__lowerCamelCase   ,       """:"""   ,       val.size()					)
       else:
            print(__lowerCamelCase   ,       """:"""   ,       __lowerCamelCase					)
  
  
  
  
  
  def    _lowerCamelCase	(	__lowerCamelCase   ,       __lowerCamelCase   ,       __lowerCamelCase   ,       __lowerCamelCase   ,       __lowerCamelCase					)   -> int:
  
       '''simple docstring'''
       UpperCAmelCase__			:     Dict										= param.size()
       if checkpoint_version == 1.0:
            # version 1.0 stores [num_heads * hidden_size * num_splits, :]
            UpperCAmelCase__			:     str										= (num_heads, hidden_size, num_splits) + input_shape[1:]
            UpperCAmelCase__			:     Any										= param.view(*__lowerCamelCase					)
            UpperCAmelCase__			:     Dict										= param.transpose(0   ,       2					)
            UpperCAmelCase__			:     str										= param.transpose(1   ,       2					).contiguous()
       elif checkpoint_version >= 2.0:
            # other versions store [num_heads * num_splits * hidden_size, :]
            UpperCAmelCase__			:     Any										= (num_heads, num_splits, hidden_size) + input_shape[1:]
            UpperCAmelCase__			:     Optional[Any]										= param.view(*__lowerCamelCase					)
            UpperCAmelCase__			:     Optional[int]										= param.transpose(0   ,       1					).contiguous()
       UpperCAmelCase__			:     Optional[Any]										= param.view(*__lowerCamelCase					)
       return param
  
  
  
  
  
  def    _lowerCamelCase	(	__lowerCamelCase   ,       __lowerCamelCase   ,       __lowerCamelCase					)   -> int:
  
       '''simple docstring'''
       UpperCAmelCase__			:     Optional[Any]										= {}
  
       # old versions did not store training args
       UpperCAmelCase__			:     int										= input_state_dict.get("""args"""   ,       __lowerCamelCase					)
       if ds_args is not None:
            # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
            # from pprint import pprint
            # pprint(vars(ds_args))
  
            UpperCAmelCase__			:     Union[str, Any]										= ds_args.tokenizer_type
            UpperCAmelCase__			:     List[str]										= ds_args.padded_vocab_size
            UpperCAmelCase__			:     Any										= ds_args.max_position_embeddings
            UpperCAmelCase__			:     Any										= ds_args.hidden_size
            UpperCAmelCase__			:     int										= ds_args.num_layers
            UpperCAmelCase__			:     int										= ds_args.num_attention_heads
            UpperCAmelCase__			:     Optional[Any]										= ds_args.ffn_hidden_size if """ffn_hidden_size""" in ds_args else 4 * ds_args.hidden_size
            # pprint(config)
  
       # The number of heads.
       UpperCAmelCase__			:     Any										= config.num_attention_heads
       # The hidden_size per head.
       UpperCAmelCase__			:     int										= config.hidden_size // heads
       # Megatron-LM checkpoint version
       if "checkpoint_version" in input_state_dict.keys():
            UpperCAmelCase__			:     str										= input_state_dict["""checkpoint_version"""]
       else:
            UpperCAmelCase__			:     Any										= 0.0
  
       # The model.
       UpperCAmelCase__			:     str										= input_state_dict["""model"""]
       # The language model.
       UpperCAmelCase__			:     str										= model["""language_model"""]
       # The embeddings.
       UpperCAmelCase__			:     List[str]										= lm["""embedding"""]
  
       # The word embeddings.
       UpperCAmelCase__			:     int										= embeddings["""word_embeddings"""]["""weight"""]
       # Truncate the embedding table to vocab_size rows.
       UpperCAmelCase__			:     Optional[int]										= word_embeddings[: config.vocab_size, :]
       # Store the word embeddings.
       UpperCAmelCase__			:     List[Any]										= word_embeddings
  
       # The position embeddings.
       UpperCAmelCase__			:     Optional[Any]										= embeddings["""position_embeddings"""]["""weight"""]
       assert pos_embeddings.size(0					) == config.max_position_embeddings and pos_embeddings.size(1					) == config.hidden_size
       # Store the position embeddings.
       UpperCAmelCase__			:     Any										= pos_embeddings
  
       # The token-type embeddings.
       UpperCAmelCase__			:     Tuple										= embeddings["""tokentype_embeddings"""]["""weight"""]
       # Store the position embeddings.
       UpperCAmelCase__			:     int										= tokentype_embeddings
  
       # The transformer.
       UpperCAmelCase__			:     Optional[Any]										= lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]
  
       # The regex to extract layer names.
       UpperCAmelCase__			:     List[Any]										= re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""					)
  
       # The simple map of names for "automated" rules.
       UpperCAmelCase__			:     Any										= {
           """attention.dense""": """.attention.output.dense.""",
           """self_attention.dense""": """.attention.output.dense.""",
           """mlp.dense_h_to_4h""": """.intermediate.dense.""",
           """mlp.dense_4h_to_h""": """.output.dense.""",
       }
  
       # Keep track of the attention/query/value tensor.
       UpperCAmelCase__			:     Optional[Any]										= None
  
       # Extract the layers.
       for key, val in transformer.items():
            # Match the name.
            UpperCAmelCase__			:     str										= layer_re.match(__lowerCamelCase					)
  
            # Stop if that's not a layer
            if m is None:
                 break
  
            # The index of the layer.
            UpperCAmelCase__			:     Any										= int(m.group(1					)					)
            # The name of the operation.
            UpperCAmelCase__			:     Optional[Any]										= m.group(2					)
            # Is it a weight or a bias?
            UpperCAmelCase__			:     Dict										= m.group(3					)
  
            # The name of the layer.
            UpperCAmelCase__			:     str										= F"bert.encoder.layer.{layer_idx}"
  
            # For layernorm(s), simply store the layer norm.
            if op_name.endswith("""layernorm"""					):
                 UpperCAmelCase__			:     Optional[int]										= """attention.ln""" if op_name.startswith("""input"""					) else """ln"""
                 UpperCAmelCase__			:     Any										= val
  
            # Transpose the QKV matrix.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "weight":
                 # Make sure the QKV pointer is nil.
                 assert attention_qkv_weight is None, ""
  
                 UpperCAmelCase__			:     Tuple										= fix_query_key_value_ordering(__lowerCamelCase   ,       __lowerCamelCase   ,       3   ,       __lowerCamelCase   ,       __lowerCamelCase					)
                 # Store the tensor as we need the bias as well to interleave QKV and biases.
                 UpperCAmelCase__			:     Tuple										= out_val
  
            # Transpose the bias.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "bias":
                 # Make sure we read the weight tensor.
                 assert attention_qkv_weight is not None, ""
  
                 # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
                 UpperCAmelCase__			:     Optional[Any]										= attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
                 UpperCAmelCase__			:     Dict										= attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
                 UpperCAmelCase__			:     Optional[Any]										= attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]
  
                 UpperCAmelCase__			:     Tuple										= fix_query_key_value_ordering(__lowerCamelCase   ,       __lowerCamelCase   ,       3   ,       __lowerCamelCase   ,       __lowerCamelCase					)
                 # Split the bias.
                 UpperCAmelCase__			:     List[Any]										= out_val[0 * config.hidden_size : 1 * config.hidden_size]
                 UpperCAmelCase__			:     Optional[Any]										= out_val[1 * config.hidden_size : 2 * config.hidden_size]
                 UpperCAmelCase__			:     Any										= out_val[2 * config.hidden_size : 3 * config.hidden_size]
  
                 # Store.
                 UpperCAmelCase__			:     str										= q
                 UpperCAmelCase__			:     Dict										= q_bias
                 UpperCAmelCase__			:     Any										= k
                 UpperCAmelCase__			:     Any										= k_bias
                 UpperCAmelCase__			:     Optional[int]										= v
                 UpperCAmelCase__			:     Tuple										= v_bias
  
                 # Clear the stored tensor.
                 UpperCAmelCase__			:     Optional[Any]										= None
  
            # Copy weights and biases as is.
            elif weight_or_bias in ["weight", "bias"]:
                 UpperCAmelCase__			:     List[Any]										= megatron_to_transformers[op_name]
                 UpperCAmelCase__			:     Union[str, Any]										= val
  
      # The final layernorm.
       UpperCAmelCase__			:     List[Any]										= transformer["""final_layernorm.weight"""]
       UpperCAmelCase__			:     List[str]										= transformer["""final_layernorm.bias"""]
  
       # The pooler.
       UpperCAmelCase__			:     List[str]										= lm["""pooler"""]
  
       # Store the matrix and the bias.
       UpperCAmelCase__			:     List[Any]										= pooler["""dense.weight"""]
       UpperCAmelCase__			:     Tuple										= pooler["""dense.bias"""]
  
       # The LM head from Megatron (for RACE).
       UpperCAmelCase__			:     Dict										= model["""lm_head"""]
  
       # The transform matrix.
       UpperCAmelCase__			:     str										= lm_head["""dense.weight"""]
       UpperCAmelCase__			:     str										= lm_head["""dense.bias"""]
  
       # The transform LN.
       UpperCAmelCase__			:     List[Any]										= lm_head["""layernorm.weight"""]
       UpperCAmelCase__			:     int										= lm_head["""layernorm.bias"""]
  
       # For the decoder, we replicate the weights.
       UpperCAmelCase__			:     Dict										= word_embeddings
       UpperCAmelCase__			:     Tuple										= lm_head["""bias"""]
  
       # The classifier from Megatron (for MLNI).
       UpperCAmelCase__			:     Optional[int]										= model["""binary_head"""]
  
       # Store the classifier.
       UpperCAmelCase__			:     Optional[int]										= binary_head["""weight"""]
       UpperCAmelCase__			:     Optional[int]										= binary_head["""bias"""]
  
       # It should be done!
       return output_state_dict
  
  
  
  
  
  def    _lowerCamelCase	(	)   -> Union[str, Any]:
  
       '''simple docstring'''
       UpperCAmelCase__			:     Optional[int]										= argparse.ArgumentParser()
       parser.add_argument("""--print-checkpoint-structure"""   ,       action="""store_true"""					)
       parser.add_argument("""path_to_checkpoint"""   ,       type=__lowerCamelCase   ,       help="""Path to the ZIP file containing the checkpoint"""					)
       parser.add_argument(
           """--config_file"""   ,       default=""""""   ,       type=__lowerCamelCase   ,       help="""An optional config json file describing the pre-trained model."""   ,       )
       UpperCAmelCase__			:     Optional[int]										= parser.parse_args()
  
       # Extract the basename.
       UpperCAmelCase__			:     Union[str, Any]										= os.path.dirname(args.path_to_checkpoint					)
  
       # Load the model.
       # the .zip is very optional, let's keep it for backward compatibility
       print(F"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""					)
       if args.path_to_checkpoint.endswith(""".zip"""					):
            with zipfile.ZipFile(args.path_to_checkpoint   ,       """r"""					) as checkpoint:
                 with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""					) as pytorch_dict:
                      UpperCAmelCase__			:     List[str]										= torch.load(__lowerCamelCase   ,       map_location="""cpu"""					)
       else:
            UpperCAmelCase__			:     Union[str, Any]										= torch.load(args.path_to_checkpoint   ,       map_location="""cpu"""					)
  
       if args.config_file == "":
            # Default config of megatron-bert 345m
            UpperCAmelCase__			:     Tuple										= MegatronBertConfig()
  
            # different megatron-bert-*-345m models have different vocab sizes, so override the default
            # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
            UpperCAmelCase__			:     List[Any]										= input_state_dict["""model"""]["""lm_head"""]["""bias"""].numel()
       else:
            UpperCAmelCase__			:     Optional[Any]										= MegatronBertConfig.from_json_file(args.config_file					)
  
       # Convert.
       print("""Converting"""					)
       UpperCAmelCase__			:     List[str]										= convert_megatron_checkpoint(__lowerCamelCase   ,       __lowerCamelCase   ,       __lowerCamelCase					)
  
       # Print the structure of converted state dict.
       if args.print_checkpoint_structure:
            recursive_print(__lowerCamelCase   ,       __lowerCamelCase					)
  
       # Store the config to file.
       print("""Saving config"""					)
       config.save_pretrained(__lowerCamelCase					)
  
       # Store the state_dict to file.
       UpperCAmelCase__			:     Dict										= os.path.join(__lowerCamelCase   ,       """pytorch_model.bin"""					)
       print(F"Saving checkpoint to \"{output_checkpoint_file}\""					)
       torch.save(__lowerCamelCase   ,       __lowerCamelCase					)
  
  
  ####################################################################################################
  
  if __name__ == "__main__":
        main()
  
  ####################################################################################################
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		__lowerCamelCase       ,			unittest.TestCase		):
       __lowerCamelCase										=     PreTrainedTokenizerFast
       __lowerCamelCase										=     False
       __lowerCamelCase										=     True
       __lowerCamelCase										=     'tokenizer_file'
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Union[str, Any]										= False  # because we don't have pretrained_vocab_files_map
            super().setUp()
            UpperCAmelCase__			:     Dict										= True
  
            UpperCAmelCase__			:     int										= ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
            UpperCAmelCase__			:     List[Any]										= """SaulLu/dummy-tokenizer-bytelevel-bpe"""
  
            # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
            UpperCAmelCase__			:     int										= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
            UpperCAmelCase__			:     Union[str, Any]										= PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
            tokenizer.save_pretrained(self.tmpdirname		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Optional[int]										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     str										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     Optional[Any]										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer_with_special_tokens_change()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)
  
            UpperCAmelCase__			:     Dict										= ("""a""" for _ in range(1000		))
            UpperCAmelCase__			:     int										= tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase  ,	length=1000  ,	vocab_size=50		)
  
            UpperCAmelCase__			:     Optional[Any]										= new_tokenizer.encode("""a🤗"""		)
            self.assertEqual(_lowerCAmelCase  ,	[64, 172, 253, 97, 245]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     List[str]										= AutoTokenizer.from_pretrained("""bert-base-cased"""		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            UpperCAmelCase__			:     Optional[Any]										= len(_lowerCAmelCase		)
  
            with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 UpperCAmelCase__			:     Tuple										= ["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 json.dump(_lowerCAmelCase  ,	open(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	"""w"""		)		)
  
                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 UpperCAmelCase__			:     Optional[int]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		) + 1		)
                 UpperCAmelCase__			:     Any										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	os.path.join(_lowerCAmelCase  ,	"""tokenizer.42.0.0.json"""		)		)
                 UpperCAmelCase__			:     Union[str, Any]										= ["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 UpperCAmelCase__			:     Optional[Any]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		)		)
                 UpperCAmelCase__			:     List[Any]										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
            UpperCAmelCase__			:     List[str]										= """hf-internal-testing/test-two-tokenizers"""
  
            # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
            UpperCAmelCase__			:     int										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28997		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
            # Testing an older version by monkey-patching the version in the module it's used.
            import transformers as old_transformers
  
            UpperCAmelCase__			:     Any										= """3.0.0"""
            UpperCAmelCase__			:     Optional[int]										= old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28996		)
            UpperCAmelCase__			:     Optional[int]										= json.loads(old_tokenizer._tokenizer.to_str()		)
            self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # See https://github.com/huggingface/transformers/pull/12550
            # and https://github.com/huggingface/tokenizers/issues/537
            UpperCAmelCase__			:     int										= PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""		)
            UpperCAmelCase__			:     List[str]										= """The Matrix is a 1999 science fiction action film."""
  
            with concurrent.futures.ThreadPoolExecutor() as executor:
                 UpperCAmelCase__			:     List[str]										= [executor.submit(self.fetch  ,	_lowerCAmelCase  ,	_lowerCAmelCase		) for i in range(10		)]
                 UpperCAmelCase__			:     Dict										= [future.result() for future in futures]
                 self.assertEqual(_lowerCAmelCase  ,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            return tokenizer.encode(_lowerCAmelCase  ,	truncation="""longest_first"""  ,	padding="""longest"""		)
  
  
  
  
  
  
  