code: 
  
  
  '''simple docstring'''
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  @require_tokenizers
  class 						__lowerCAmelCase		(lowerCamelCase__						,  unittest.TestCase  ):
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        lowerCAmelCase__      :  Dict						      =							PreTrainedTokenizerFast
        lowerCAmelCase__      :  List[Any]						      =							False
        lowerCAmelCase__      :  Optional[Any]						      =							True
        lowerCAmelCase__      :  List[Any]						      =							"""tokenizer_file"""
  
  
  
        def    UpperCamelCase__	(self					:					Dict				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				False  # because we don't have pretrained_vocab_files_map
               super().setUp()
               lowercase__       =				True
  
               lowercase__       =				['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
               lowercase__       =				'''SaulLu/dummy-tokenizer-bytelevel-bpe'''
  
               # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
               lowercase__       =				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
               lowercase__       =				PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
               tokenizer.save_pretrained(self.tmpdirname				)
  
  
  
        def    UpperCamelCase__	(self					:					List[str]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               pass
  
  
  
        def    UpperCamelCase__	(self					:					List[str]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               pass
  
  
  
        def    UpperCamelCase__	(self					:					Dict				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               pass
  
  
  
        def    UpperCamelCase__	(self					:					str				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               pass
  
  
  
        def    UpperCamelCase__	(self					:					List[str]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				self.tmpdirname
               # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
               for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                      with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                             try:
                                    lowercase__       =				tempfile.mkdtemp()
                                    lowercase__       =				self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_     ,		**SCREAMING_SNAKE_CASE_				)
  
                                    tokenizer.save_pretrained(self.tmpdirname				)
                                    super().test_training_new_tokenizer()
                             finally:
                                    # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                                    # is restored
                                    shutil.rmtree(self.tmpdirname				)
                                    lowercase__       =				tmpdirname_orig
  
  
  
        def    UpperCamelCase__	(self					:					Tuple				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				self.tmpdirname
               # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
               for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                      with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                             try:
                                    lowercase__       =				tempfile.mkdtemp()
                                    lowercase__       =				self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_     ,		**SCREAMING_SNAKE_CASE_				)
  
                                    tokenizer.save_pretrained(self.tmpdirname				)
                                    super().test_training_new_tokenizer_with_special_tokens_change()
                             finally:
                                    # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                                    # is restored
                                    shutil.rmtree(self.tmpdirname				)
                                    lowercase__       =				tmpdirname_orig
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					List[str]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)
  
               lowercase__       =				('''a''' for _ in range(1000				))
               lowercase__       =				tokenizer.train_new_from_iterator(text_iterator=SCREAMING_SNAKE_CASE_     ,		length=1000     ,		vocab_size=50				)
  
               lowercase__       =				new_tokenizer.encode('''a🤗'''				)
               self.assertEqual(SCREAMING_SNAKE_CASE_     ,		[64, 172, 253, 97, 245]				)
  
  
  
  
  
  
  @require_tokenizers
  class 						__lowerCAmelCase		(unittest.TestCase  ):
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					Optional[int]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				AutoTokenizer.from_pretrained('''bert-base-cased'''				)
               lowercase__       =				json.loads(tokenizer._tokenizer.to_str()				)
               lowercase__       =				len(SCREAMING_SNAKE_CASE_				)
  
               with tempfile.TemporaryDirectory() as tmp_dir:
                      # Hack to save this in the tokenizer_config.json
                      lowercase__       =				['''tokenizer.4.0.0.json''']
                      tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_				)
                      json.dump(SCREAMING_SNAKE_CASE_     ,		open(os.path.join(SCREAMING_SNAKE_CASE_     ,		'''tokenizer.4.0.0.json'''				)     ,		'''w'''				)				)
  
                      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                      lowercase__       =				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_				)
                      self.assertEqual(len(SCREAMING_SNAKE_CASE_				)     ,		len(SCREAMING_SNAKE_CASE_				) + 1				)
                      lowercase__       =				json.loads(new_tokenizer._tokenizer.to_str()				)
                      self.assertIn('''huggingface'''     ,		json_tokenizer['''model''']['''vocab''']				)
  
                      # Will need to be adjusted if we reach v42 and this test is still here.
                      # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                      shutil.move(os.path.join(SCREAMING_SNAKE_CASE_     ,		'''tokenizer.4.0.0.json'''				)     ,		os.path.join(SCREAMING_SNAKE_CASE_     ,		'''tokenizer.42.0.0.json'''				)				)
                      lowercase__       =				['''tokenizer.42.0.0.json''']
                      tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_				)
                      lowercase__       =				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_				)
                      self.assertEqual(len(SCREAMING_SNAKE_CASE_				)     ,		len(SCREAMING_SNAKE_CASE_				)				)
                      lowercase__       =				json.loads(new_tokenizer._tokenizer.to_str()				)
                      self.assertNotIn('''huggingface'''     ,		json_tokenizer['''model''']['''vocab''']				)
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					str				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				'''hf-internal-testing/test-two-tokenizers'''
  
               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               lowercase__       =				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_				)
               self.assertEqual(len(SCREAMING_SNAKE_CASE_				)     ,		28997				)
               lowercase__       =				json.loads(tokenizer._tokenizer.to_str()				)
               self.assertIn('''huggingface'''     ,		json_tokenizer['''model''']['''vocab''']				)
  
               # Testing an older version by monkey-patching the version in the module it's used.
               import transformers as old_transformers
  
               lowercase__       =				'''3.0.0'''
               lowercase__       =				old_transformers.models.auto.AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_				)
               self.assertEqual(len(SCREAMING_SNAKE_CASE_				)     ,		28996				)
               lowercase__       =				json.loads(old_tokenizer._tokenizer.to_str()				)
               self.assertNotIn('''huggingface'''     ,		json_tokenizer['''model''']['''vocab''']				)
  
  
  
  
  
  
  @require_tokenizers
  class 						__lowerCAmelCase		(unittest.TestCase  ):
  
  
  
  
  
        '''simple docstring'''
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					List[Any]				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               lowercase__       =				PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''				)
               lowercase__       =				'''The Matrix is a 1999 science fiction action film.'''
  
               with concurrent.futures.ThreadPoolExecutor() as executor:
                      lowercase__       =				[executor.submit(self.fetch     ,		SCREAMING_SNAKE_CASE_     ,		SCREAMING_SNAKE_CASE_				) for i in range(10				)]
                      lowercase__       =				[future.result() for future in futures]
                      self.assertEqual(SCREAMING_SNAKE_CASE_     ,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)
  
  
  
  
  
  
        def    UpperCamelCase__	(self					:					Tuple     ,		UpperCamelCase					:					Any     ,		UpperCamelCase					:					Any				):
  
               '''simple docstring'''
  
  
  
  
  
  
  
               return tokenizer.encode(SCREAMING_SNAKE_CASE_     ,		truncation='''longest_first'''     ,		padding='''longest'''				)
  
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  import argparse
  
  import torch
  
  from transformers import ElectraConfig, ElectraForMaskedLM, ElectraForPreTraining, load_tf_weights_in_electra
  from transformers.utils import logging
  
  
  logging.set_verbosity_info()
  
  def _SCREAMING_SNAKE_CASE   (A						,				A						,				A						,				A ) -> Dict:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				ElectraConfig.from_json_file(A )
         print(f"Building PyTorch model from configuration: {config}" )
  
         if discriminator_or_generator == "discriminator":
                lowercase__       =				ElectraForPreTraining(A )
         elif discriminator_or_generator == "generator":
                lowercase__       =				ElectraForMaskedLM(A )
         else:
                raise ValueError('''The discriminator_or_generator argument should be either \'discriminator\' or \'generator\'''' )
  
         # Load weights from tf checkpoint
         load_tf_weights_in_electra(
             A						,				A						,				A						,				discriminator_or_generator=A )
  
         # Save pytorch-model
         print(f"Save PyTorch model to {pytorch_dump_path}" )
         torch.save(model.state_dict()						,				A )
  
  
  if __name__ == "__main__":
   lowerCamelCase							:     Tuple								=			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
   )
   parser.add_argument(
       '--config_file',
       default=None,
       type=str,
       required=True,
       help='The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.',
   )
   parser.add_argument(
       '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
   )
   parser.add_argument(
       '--discriminator_or_generator',
       default=None,
       type=str,
       required=True,
       help=(
           'Whether to export the generator or the discriminator. Should be a string, either \'discriminator\' or '
           '\'generator\'.'
       ),
   )
   lowerCamelCase							:     Optional[int]								=			parser.parse_args()
   convert_tf_checkpoint_to_pytorch(
       args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path, args.discriminator_or_generator
   )
  
  
  
  
  