





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class A_      (    _lowerCamelCase				,   unittest.TestCase     ):
   lowerCAmelCase__       =				PreTrainedTokenizerFast
   lowerCAmelCase__       =				False
   lowerCAmelCase__       =				True
   lowerCAmelCase__       =				"""tokenizer_file"""




   def _lowerCAmelCase   (self			:Optional[Any]				)->      List[str]:
         __A      =       False  # because we don't have pretrained_vocab_files_map
         super().setUp()
         __A      =       True

         __A      =       ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
         __A      =       '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

         # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
         __A      =       [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

         __A      =       PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
         tokenizer.save_pretrained(self.tmpdirname				)




   def _lowerCAmelCase   (self			:int				)->      Dict:
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass




   def _lowerCAmelCase   (self			:Union[str, Any]				)->      Union[str, Any]:
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass




   def _lowerCAmelCase   (self			:Union[str, Any]				)->      Union[str, Any]:
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass




   def _lowerCAmelCase   (self			:Optional[int]				)->      int:
         # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
         pass




   def _lowerCAmelCase   (self			:Dict				)->      List[Any]:
         __A      =       self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                     try:
                           __A      =       tempfile.mkdtemp()
                           __A      =       self.rust_tokenizer_class.from_pretrained(_UpperCamelCase					,     **_UpperCamelCase				)

                           tokenizer.save_pretrained(self.tmpdirname				)
                           super().test_training_new_tokenizer()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname				)
                           __A      =       tmpdirname_orig




   def _lowerCAmelCase   (self			:List[str]				)->      str:
         __A      =       self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                     try:
                           __A      =       tempfile.mkdtemp()
                           __A      =       self.rust_tokenizer_class.from_pretrained(_UpperCamelCase					,     **_UpperCamelCase				)

                           tokenizer.save_pretrained(self.tmpdirname				)
                           super().test_training_new_tokenizer_with_special_tokens_change()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname				)
                           __A      =       tmpdirname_orig






   def _lowerCAmelCase   (self			:List[Any]				)->      List[str]:
         __A      =       self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

         __A      =       ('''a''' for _ in range(1000				))
         __A      =       tokenizer.train_new_from_iterator(text_iterator=_UpperCamelCase					,     length=1000					,     vocab_size=50				)

         __A      =       new_tokenizer.encode('''a🤗'''				)
         self.assertEqual(_UpperCamelCase					,     [64, 172, 253, 97, 245]				)





@require_tokenizers
class A_      (    unittest.TestCase     ):




   def _lowerCAmelCase   (self			:Union[str, Any]				)->      Tuple:
         __A      =       AutoTokenizer.from_pretrained('''bert-base-cased'''				)
         __A      =       json.loads(tokenizer._tokenizer.to_str()				)
         __A      =       len(_UpperCamelCase				)

         with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               __A      =       ['''tokenizer.4.0.0.json''']
               tokenizer.save_pretrained(_UpperCamelCase				)
               json.dump(_UpperCamelCase					,     open(os.path.join(_UpperCamelCase					,     '''tokenizer.4.0.0.json'''				)					,     '''w'''				)				)

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               __A      =       AutoTokenizer.from_pretrained(_UpperCamelCase				)
               self.assertEqual(len(_UpperCamelCase				)					,     len(_UpperCamelCase				) + 1				)
               __A      =       json.loads(new_tokenizer._tokenizer.to_str()				)
               self.assertIn('''huggingface'''					,     json_tokenizer['''model''']['''vocab''']				)

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(_UpperCamelCase					,     '''tokenizer.4.0.0.json'''				)					,     os.path.join(_UpperCamelCase					,     '''tokenizer.42.0.0.json'''				)				)
               __A      =       ['''tokenizer.42.0.0.json''']
               tokenizer.save_pretrained(_UpperCamelCase				)
               __A      =       AutoTokenizer.from_pretrained(_UpperCamelCase				)
               self.assertEqual(len(_UpperCamelCase				)					,     len(_UpperCamelCase				)				)
               __A      =       json.loads(new_tokenizer._tokenizer.to_str()				)
               self.assertNotIn('''huggingface'''					,     json_tokenizer['''model''']['''vocab''']				)






   def _lowerCAmelCase   (self			:List[Any]				)->      List[str]:
         # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
         __A      =       '''hf-internal-testing/test-two-tokenizers'''

         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         __A      =       AutoTokenizer.from_pretrained(_UpperCamelCase				)
         self.assertEqual(len(_UpperCamelCase				)					,     2_8997				)
         __A      =       json.loads(tokenizer._tokenizer.to_str()				)
         self.assertIn('''huggingface'''					,     json_tokenizer['''model''']['''vocab''']				)

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         __A      =       '''3.0.0'''
         __A      =       old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCamelCase				)
         self.assertEqual(len(_UpperCamelCase				)					,     2_8996				)
         __A      =       json.loads(old_tokenizer._tokenizer.to_str()				)
         self.assertNotIn('''huggingface'''					,     json_tokenizer['''model''']['''vocab''']				)





@require_tokenizers
class A_      (    unittest.TestCase     ):




   def _lowerCAmelCase   (self			:int				)->      Union[str, Any]:
         # See https://github.com/huggingface/transformers/pull/12550
         # and https://github.com/huggingface/tokenizers/issues/537
         __A      =       PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''				)
         __A      =       '''The Matrix is a 1999 science fiction action film.'''

         with concurrent.futures.ThreadPoolExecutor() as executor:
               __A      =       [executor.submit(self.fetch					,     _UpperCamelCase					,     _UpperCamelCase				) for i in range(10				)]
               __A      =       [future.result() for future in futures]
               self.assertEqual(_UpperCamelCase					,     [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)






   def _lowerCAmelCase   (self			:Any					,     _UpperCamelCase			:Union[str, Any]					,     _UpperCamelCase			:Union[str, Any]				)->      str:
         return tokenizer.encode(_UpperCamelCase					,     truncation='''longest_first'''					,     padding='''longest'''				)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


snake_case__      :							Optional[int]									=  {'configuration_cvt': ['CVT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'CvtConfig']}


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__      :							Union[str, Any]									=  [
           'CVT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'CvtForImageClassification',
           'CvtModel',
           'CvtPreTrainedModel',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__      :							List[str]									=  [
           'TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'TFCvtForImageClassification',
           'TFCvtModel',
           'TFCvtPreTrainedModel',
       ]

if TYPE_CHECKING:
       from .configuration_cvt import CVT_PRETRAINED_CONFIG_ARCHIVE_MAP, CvtConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_cvt import (
                  CVT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  CvtForImageClassification,
                  CvtModel,
                  CvtPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_cvt import (
                  TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFCvtForImageClassification,
                  TFCvtModel,
                  TFCvtPreTrainedModel,
              )


else:
       import sys

       snake_case__      :							Dict									=  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





