code: 
  
  
  
  
  
  
  import torch
  from torch import autograd
  
  
  
  
  
  
  
  class 							UpperCAmelCase_     (		autograd.Function		):
  
  
  
  
       @staticmethod
       def      __UpperCAmelCase     (							_lowerCAmelCase  ,	_lowerCAmelCase  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            UpperCAmelCase__			:     Optional[int]										= inputs.numel()
            UpperCAmelCase__			:     List[Any]										= int(0.0_0_5 * nb_elems		) + 1
            if sigmoid:
                 UpperCAmelCase__			:     Union[str, Any]										= (torch.sigmoid(SCREAMING_SNAKE_CASE_		) > threshold).type(inputs.type()		)
            else:
                 UpperCAmelCase__			:     int										= (inputs > threshold).type(inputs.type()		)
            if mask.sum() < nb_min:
                 # We limit the pruning so that at least 0.5% (half a percent) of the weights are remaining
                 UpperCAmelCase__			:     List[str]										= inputs.flatten().kthvalue(max(nb_elems - nb_min  ,	1		)		).values
                 UpperCAmelCase__			:     Tuple										= (inputs > k_threshold).type(inputs.type()		)
            return mask
  
  
  
  
       @staticmethod
       def      __UpperCAmelCase     (							_lowerCAmelCase  ,	_lowerCAmelCase		):
            return gradOutput, None, None
  
  
  
  
  
  
  
  class 							UpperCAmelCase_     (		autograd.Function		):
  
  
  
  
       @staticmethod
       def      __UpperCAmelCase     (							_lowerCAmelCase  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            UpperCAmelCase__			:     Any										= inputs.clone()
            UpperCAmelCase__			,		UpperCAmelCase__			:     str										= inputs.flatten().sort(descending=SCREAMING_SNAKE_CASE_		)
            UpperCAmelCase__			:     Optional[int]										= int(threshold * inputs.numel()		)
  
            # flat_out and mask access the same memory.
            UpperCAmelCase__			:     List[str]										= mask.flatten()
            UpperCAmelCase__			:     int										= 0
            UpperCAmelCase__			:     str										= 1
            return mask
  
  
  
  
       @staticmethod
       def      __UpperCAmelCase     (							_lowerCAmelCase  ,	_lowerCAmelCase		):
            return gradOutput, None
  
  
  
  
  
  
  
  class 							UpperCAmelCase_     (		lowerCamelCase__		):
  
  
  
  
       @staticmethod
       def      __UpperCAmelCase     (							_lowerCAmelCase  ,	_lowerCAmelCase		):
            UpperCAmelCase__			:     Union[str, Any]										= inputs.clone()
            UpperCAmelCase__			,		UpperCAmelCase__			:     Optional[Any]										= inputs.abs().flatten().sort(descending=SCREAMING_SNAKE_CASE_		)
            UpperCAmelCase__			:     List[str]										= int(threshold * inputs.numel()		)
  
            # flat_out and mask access the same memory.
            UpperCAmelCase__			:     Optional[int]										= mask.flatten()
            UpperCAmelCase__			:     Dict										= 0
            UpperCAmelCase__			:     List[str]										= 1
            return mask
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		__lowerCamelCase       ,			unittest.TestCase		):
       __lowerCamelCase										=     PreTrainedTokenizerFast
       __lowerCamelCase										=     False
       __lowerCamelCase										=     True
       __lowerCamelCase										=     'tokenizer_file'
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Union[str, Any]										= False  # because we don't have pretrained_vocab_files_map
            super().setUp()
            UpperCAmelCase__			:     Dict										= True
  
            UpperCAmelCase__			:     int										= ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
            UpperCAmelCase__			:     List[Any]										= """SaulLu/dummy-tokenizer-bytelevel-bpe"""
  
            # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
            UpperCAmelCase__			:     int										= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
            UpperCAmelCase__			:     Union[str, Any]										= PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
            tokenizer.save_pretrained(self.tmpdirname		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Optional[int]										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     str										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     Optional[Any]										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer_with_special_tokens_change()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)
  
            UpperCAmelCase__			:     Dict										= ("""a""" for _ in range(1000		))
            UpperCAmelCase__			:     int										= tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase  ,	length=1000  ,	vocab_size=50		)
  
            UpperCAmelCase__			:     Optional[Any]										= new_tokenizer.encode("""a🤗"""		)
            self.assertEqual(_lowerCAmelCase  ,	[64, 172, 253, 97, 245]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     List[str]										= AutoTokenizer.from_pretrained("""bert-base-cased"""		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            UpperCAmelCase__			:     Optional[Any]										= len(_lowerCAmelCase		)
  
            with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 UpperCAmelCase__			:     Tuple										= ["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 json.dump(_lowerCAmelCase  ,	open(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	"""w"""		)		)
  
                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 UpperCAmelCase__			:     Optional[int]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		) + 1		)
                 UpperCAmelCase__			:     Any										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	os.path.join(_lowerCAmelCase  ,	"""tokenizer.42.0.0.json"""		)		)
                 UpperCAmelCase__			:     Union[str, Any]										= ["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 UpperCAmelCase__			:     Optional[Any]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		)		)
                 UpperCAmelCase__			:     List[Any]										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
            UpperCAmelCase__			:     List[str]										= """hf-internal-testing/test-two-tokenizers"""
  
            # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
            UpperCAmelCase__			:     int										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28997		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
            # Testing an older version by monkey-patching the version in the module it's used.
            import transformers as old_transformers
  
            UpperCAmelCase__			:     Any										= """3.0.0"""
            UpperCAmelCase__			:     Optional[int]										= old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28996		)
            UpperCAmelCase__			:     Optional[int]										= json.loads(old_tokenizer._tokenizer.to_str()		)
            self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # See https://github.com/huggingface/transformers/pull/12550
            # and https://github.com/huggingface/tokenizers/issues/537
            UpperCAmelCase__			:     int										= PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""		)
            UpperCAmelCase__			:     List[str]										= """The Matrix is a 1999 science fiction action film."""
  
            with concurrent.futures.ThreadPoolExecutor() as executor:
                 UpperCAmelCase__			:     List[str]										= [executor.submit(self.fetch  ,	_lowerCAmelCase  ,	_lowerCAmelCase		) for i in range(10		)]
                 UpperCAmelCase__			:     Dict										= [future.result() for future in futures]
                 self.assertEqual(_lowerCAmelCase  ,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            return tokenizer.encode(_lowerCAmelCase  ,	truncation="""longest_first"""  ,	padding="""longest"""		)
  
  
  
  
  
  
  