code: 
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  @require_tokenizers
  class   __lowerCamelCase (_a							,			unittest.TestCase	):
         _lowercase       							=    PreTrainedTokenizerFast
         _lowercase       							=    False
         _lowercase       							=    True
         _lowercase       							=    """tokenizer_file"""
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[int]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = False  # because we don't have pretrained_vocab_files_map
              super().setUp()
              __UpperCamelCase   = True
  
              __UpperCamelCase   = ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
              __UpperCamelCase   = 'SaulLu/dummy-tokenizer-bytelevel-bpe'
  
              # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
              __UpperCamelCase   = [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
              __UpperCamelCase   = PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
              tokenizer.save_pretrained(self.tmpdirname				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              pass
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   List[Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              pass
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Dict				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              pass
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              pass
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Any				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = self.tmpdirname
              # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                        try:
                             __UpperCamelCase   = tempfile.mkdtemp()
                             __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(A_,**A_				)
  
                             tokenizer.save_pretrained(self.tmpdirname				)
                             super().test_training_new_tokenizer()
                        finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname				)
                             __UpperCamelCase   = tmpdirname_orig
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Any				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = self.tmpdirname
              # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                        try:
                             __UpperCamelCase   = tempfile.mkdtemp()
                             __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(A_,**A_				)
  
                             tokenizer.save_pretrained(self.tmpdirname				)
                             super().test_training_new_tokenizer_with_special_tokens_change()
                        finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname				)
                             __UpperCamelCase   = tmpdirname_orig
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)
  
              __UpperCamelCase   = ('a' for _ in range(1000				))
              __UpperCamelCase   = tokenizer.train_new_from_iterator(text_iterator=A_,length=1000,vocab_size=50				)
  
              __UpperCamelCase   = new_tokenizer.encode('a🤗'				)
              self.assertEqual(A_,[64, 172, 253, 97, 245]				)
  
  
  
  @require_tokenizers
  class   __lowerCamelCase (unittest.TestCase	):
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = AutoTokenizer.from_pretrained('bert-base-cased'				)
              __UpperCamelCase   = json.loads(tokenizer._tokenizer.to_str()				)
              __UpperCamelCase   = len(A_				)
  
              with tempfile.TemporaryDirectory() as tmp_dir:
                   # Hack to save this in the tokenizer_config.json
                   __UpperCamelCase   = ['tokenizer.4.0.0.json']
                   tokenizer.save_pretrained(A_				)
                   json.dump(A_,open(os.path.join(A_,'tokenizer.4.0.0.json'				),'w'				)				)
  
                   # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                   __UpperCamelCase   = AutoTokenizer.from_pretrained(A_				)
                   self.assertEqual(len(A_				),len(A_				) + 1				)
                   __UpperCamelCase   = json.loads(new_tokenizer._tokenizer.to_str()				)
                   self.assertIn('huggingface',json_tokenizer['model']['vocab']				)
  
                   # Will need to be adjusted if we reach v42 and this test is still here.
                   # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                   shutil.move(os.path.join(A_,'tokenizer.4.0.0.json'				),os.path.join(A_,'tokenizer.42.0.0.json'				)				)
                   __UpperCamelCase   = ['tokenizer.42.0.0.json']
                   tokenizer.save_pretrained(A_				)
                   __UpperCamelCase   = AutoTokenizer.from_pretrained(A_				)
                   self.assertEqual(len(A_				),len(A_				)				)
                   __UpperCamelCase   = json.loads(new_tokenizer._tokenizer.to_str()				)
                   self.assertNotIn('huggingface',json_tokenizer['model']['vocab']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[int]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = 'hf-internal-testing/test-two-tokenizers'
  
              # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
              __UpperCamelCase   = AutoTokenizer.from_pretrained(A_				)
              self.assertEqual(len(A_				),2_8997				)
              __UpperCamelCase   = json.loads(tokenizer._tokenizer.to_str()				)
              self.assertIn('huggingface',json_tokenizer['model']['vocab']				)
  
              # Testing an older version by monkey-patching the version in the module it's used.
              import transformers as old_transformers
  
              __UpperCamelCase   = '3.0.0'
              __UpperCamelCase   = old_transformers.models.auto.AutoTokenizer.from_pretrained(A_				)
              self.assertEqual(len(A_				),2_8996				)
              __UpperCamelCase   = json.loads(old_tokenizer._tokenizer.to_str()				)
              self.assertNotIn('huggingface',json_tokenizer['model']['vocab']				)
  
  
  
  @require_tokenizers
  class   __lowerCamelCase (unittest.TestCase	):
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[int]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'				)
              __UpperCamelCase   = 'The Matrix is a 1999 science fiction action film.'
  
              with concurrent.futures.ThreadPoolExecutor() as executor:
                   __UpperCamelCase   = [executor.submit(self.fetch,A_,A_				) for i in range(10				)]
                   __UpperCamelCase   = [future.result() for future in futures]
                   self.assertEqual(A_,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Any,A_:   List[Any],A_:   Any				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              return tokenizer.encode(A_,truncation='longest_first',padding='longest'				)
  
  
  
style_context: 
  
  
  from __future__ import annotations
  
  
  
  
  
  
  def 		_A		(		_lowercase   )				->   int | float:
  
  
       """simple docstring"""
  
  
       if len(_lowercase   ) == 0:
            raise ValueError('find_max() arg is an empty sequence'   )
       __UpperCamelCase   = nums[0]
       for x in nums:
            if x > max_num:
                 __UpperCamelCase   = x
       return max_num
  
  
  if __name__ == "__main__":
      import doctest
  
      doctest.testmod(verbose=True)
  
  
  