






"""simple docstring"""
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 						a_ (      snake_case_    ,		unittest.TestCase					):







      '''simple docstring'''

      lowerCamelCase__			:							str					=      PreTrainedTokenizerFast
      lowerCamelCase__			:							Dict					=      False
      lowerCamelCase__			:							Union[str, Any]					=      True
      lowerCamelCase__			:							List[str]					=      'tokenizer_file'




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[Any]								=		False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       lowerCamelCase__	:     List[str]								=		True

       lowerCamelCase__	:     Union[str, Any]								=		['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
       lowerCamelCase__	:     Dict								=		'SaulLu/dummy-tokenizer-bytelevel-bpe'

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       lowerCamelCase__	:     Optional[int]								=		[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       lowerCamelCase__	:     int								=		PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
       tokenizer.save_pretrained(self.tmpdirname   )




      def      a__				(self   ):

       '''simple docstring'''



       pass




      def      a__				(self   ):

       '''simple docstring'''



       pass




      def      a__				(self   ):

       '''simple docstring'''



       pass




      def      a__				(self   ):

       '''simple docstring'''



       pass




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[int]								=		self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
         try:
          lowerCamelCase__	:     Optional[Any]								=		tempfile.mkdtemp()
          lowerCamelCase__	:     Union[str, Any]								=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase_,	**lowerCamelCase_   )

          tokenizer.save_pretrained(self.tmpdirname   )
          super().test_training_new_tokenizer()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname   )
          lowerCamelCase__	:     Tuple								=		tmpdirname_orig




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[Any]								=		self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
         try:
          lowerCamelCase__	:     List[str]								=		tempfile.mkdtemp()
          lowerCamelCase__	:     List[str]								=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase_,	**lowerCamelCase_   )

          tokenizer.save_pretrained(self.tmpdirname   )
          super().test_training_new_tokenizer_with_special_tokens_change()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname   )
          lowerCamelCase__	:     Optional[Any]								=		tmpdirname_orig





      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Union[str, Any]								=		self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

       lowerCamelCase__	:     str								=		('a' for _ in range(1_0_0_0   ))
       lowerCamelCase__	:     Tuple								=		tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase_,	length=1_0_0_0,	vocab_size=5_0   )

       lowerCamelCase__	:     Tuple								=		new_tokenizer.encode('a🤗'   )
       self.assertEqual(lowerCamelCase_,	[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]   )



@require_tokenizers
class 						a_ (      unittest.TestCase					):







      '''simple docstring'''

      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Dict								=		AutoTokenizer.from_pretrained('bert-base-cased'   )
       lowerCamelCase__	:     List[str]								=		json.loads(tokenizer._tokenizer.to_str()   )
       lowerCamelCase__	:     str								=		len(lowerCamelCase_   )

       with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        lowerCamelCase__	:     Dict								=		['tokenizer.4.0.0.json']
        tokenizer.save_pretrained(lowerCamelCase_   )
        json.dump(lowerCamelCase_,	open(os.path.join(lowerCamelCase_,	'tokenizer.4.0.0.json'   ),	'w'   )   )

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        lowerCamelCase__	:     Union[str, Any]								=		AutoTokenizer.from_pretrained(lowerCamelCase_   )
        self.assertEqual(len(lowerCamelCase_   ),	len(lowerCamelCase_   ) + 1   )
        lowerCamelCase__	:     Tuple								=		json.loads(new_tokenizer._tokenizer.to_str()   )
        self.assertIn('huggingface',	json_tokenizer['model']['vocab']   )

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(lowerCamelCase_,	'tokenizer.4.0.0.json'   ),	os.path.join(lowerCamelCase_,	'tokenizer.42.0.0.json'   )   )
        lowerCamelCase__	:     Tuple								=		['tokenizer.42.0.0.json']
        tokenizer.save_pretrained(lowerCamelCase_   )
        lowerCamelCase__	:     Optional[Any]								=		AutoTokenizer.from_pretrained(lowerCamelCase_   )
        self.assertEqual(len(lowerCamelCase_   ),	len(lowerCamelCase_   )   )
        lowerCamelCase__	:     int								=		json.loads(new_tokenizer._tokenizer.to_str()   )
        self.assertNotIn('huggingface',	json_tokenizer['model']['vocab']   )





      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[Any]								=		'hf-internal-testing/test-two-tokenizers'

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       lowerCamelCase__	:     int								=		AutoTokenizer.from_pretrained(lowerCamelCase_   )
       self.assertEqual(len(lowerCamelCase_   ),	2_8_9_9_7   )
       lowerCamelCase__	:     List[str]								=		json.loads(tokenizer._tokenizer.to_str()   )
       self.assertIn('huggingface',	json_tokenizer['model']['vocab']   )

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       lowerCamelCase__	:     str								=		'3.0.0'
       lowerCamelCase__	:     str								=		old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase_   )
       self.assertEqual(len(lowerCamelCase_   ),	2_8_9_9_6   )
       lowerCamelCase__	:     Optional[int]								=		json.loads(old_tokenizer._tokenizer.to_str()   )
       self.assertNotIn('huggingface',	json_tokenizer['model']['vocab']   )



@require_tokenizers
class 						a_ (      unittest.TestCase					):







      '''simple docstring'''

      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     str								=		PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'   )
       lowerCamelCase__	:     Optional[int]								=		'The Matrix is a 1999 science fiction action film.'

       with concurrent.futures.ThreadPoolExecutor() as executor:
        lowerCamelCase__	:     List[Any]								=		[executor.submit(self.fetch,	lowerCamelCase_,	lowerCamelCase_   ) for i in range(1_0   )]
        lowerCamelCase__	:     str								=		[future.result() for future in futures]
        self.assertEqual(lowerCamelCase_,	[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0   )]   )





      def      a__				(self,	lowerCamelCase_,	lowerCamelCase_   ):

       '''simple docstring'''



       return tokenizer.encode(lowerCamelCase_,	truncation='longest_first',	padding='longest'   )









"""simple docstring"""
from transformers import HfArgumentParser, PyTorchBenchmark, PyTorchBenchmarkArguments






def 			lowerCamelCase_    (		):
 lowerCamelCase__	:     Tuple								=		HfArgumentParser(_lowerCamelCase							)
 try:
  lowerCamelCase__	:     Dict								=		parser.parse_args_into_dataclasses()[0]
 except ValueError as e:
  lowerCamelCase__	:     Optional[int]								=		'Arg --no_{0} is no longer used, please use --no-{0} instead.'
  lowerCamelCase__	:     Optional[Any]								=		' '.join(str(_lowerCamelCase							).split(' '							)[:-1]							)
  lowerCamelCase__	:     int								=		''
  lowerCamelCase__	:     Optional[int]								=		eval(str(_lowerCamelCase							).split(' '							)[-1]							)
  lowerCamelCase__	:     Tuple								=		[]
  for arg in depreciated_args:
   # arg[2:] removes '--'
   if arg[2:] in PyTorchBenchmarkArguments.deprecated_args:
    # arg[5:] removes '--no_'
    full_error_msg += arg_error_msg.format(arg[5:]							)
   else:
    wrong_args.append(_lowerCamelCase							)
  if len(_lowerCamelCase							) > 0:
   lowerCamelCase__	:     Optional[int]								=		full_error_msg + begin_error_msg + str(_lowerCamelCase							)
  raise ValueError(_lowerCamelCase							)

 lowerCamelCase__	:     Any								=		PyTorchBenchmark(args=_lowerCamelCase							)
 benchmark.run()


if __name__ == "__main__":
      main()

