




'''simple docstring'''

from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import SwinConfig
from transformers.testing_utils import require_tf, require_vision, slow, to_atuple
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers.models.swin.modeling_tf_swin import (
		    TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFSwinForImageClassification,
		    TFSwinForMaskedImageModeling,
		    TFSwinModel,
		)


if is_vision_available():
		from PIL import Image

		from transformers import AutoImageProcessor





class __magic_name__:






						def __init__(   self						:						Dict					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Optional[Any]=1_3					,   __UpperCamelCase						:						int=3_2					,   __UpperCamelCase						:						str=2					,   __UpperCamelCase						:						Optional[Any]=3					,   __UpperCamelCase						:						Union[str, Any]=1_6					,   __UpperCamelCase						:						int=[1, 2, 1]					,   __UpperCamelCase						:						Optional[int]=[2, 2, 4]					,   __UpperCamelCase						:						Any=2					,   __UpperCamelCase						:						Union[str, Any]=2.0					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Dict=0.0					,   __UpperCamelCase						:						Union[str, Any]=0.0					,   __UpperCamelCase						:						int=0.1					,   __UpperCamelCase						:						Optional[Any]="gelu"					,   __UpperCamelCase						:						Tuple=False					,   __UpperCamelCase						:						Optional[Any]=True					,   __UpperCamelCase						:						Tuple=0.02					,   __UpperCamelCase						:						List[Any]=1E-5					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Dict=None					,   __UpperCamelCase						:						Optional[int]=True					,   __UpperCamelCase						:						Dict=1_0					,   __UpperCamelCase						:						Dict=8					,   ):



									'''simple docstring'''






									snake_case__          =						parent
									snake_case__          =						batch_size
									snake_case__          =						image_size
									snake_case__          =						patch_size
									snake_case__          =						num_channels
									snake_case__          =						embed_dim
									snake_case__          =						depths
									snake_case__          =						num_heads
									snake_case__          =						window_size
									snake_case__          =						mlp_ratio
									snake_case__          =						qkv_bias
									snake_case__          =						hidden_dropout_prob
									snake_case__          =						attention_probs_dropout_prob
									snake_case__          =						drop_path_rate
									snake_case__          =						hidden_act
									snake_case__          =						use_absolute_embeddings
									snake_case__          =						patch_norm
									snake_case__          =						layer_norm_eps
									snake_case__          =						initializer_range
									snake_case__          =						is_training
									snake_case__          =						scope
									snake_case__          =						use_labels
									snake_case__          =						type_sequence_label_size
									snake_case__          =						encoder_stride






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]       )

									snake_case__          =						None
									if self.use_labels:
												snake_case__          =						ids_tensor([self.batch_size]					,   self.type_sequence_label_size       )

									snake_case__          =						self.get_config()

									return config, pixel_values, labels






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return SwinConfig(
									    image_size=self.image_size					,   patch_size=self.patch_size					,   num_channels=self.num_channels					,   embed_dim=self.embed_dim					,   depths=self.depths					,   num_heads=self.num_heads					,   window_size=self.window_size					,   mlp_ratio=self.mlp_ratio					,   qkv_bias=self.qkv_bias					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   drop_path_rate=self.drop_path_rate					,   hidden_act=self.hidden_act					,   use_absolute_embeddings=self.use_absolute_embeddings					,   path_norm=self.patch_norm					,   layer_norm_eps=self.layer_norm_eps					,   initializer_range=self.initializer_range					,   encoder_stride=self.encoder_stride					,   )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						TFSwinModel(config=__UpperCamelCase       )
									snake_case__          =						model(__UpperCamelCase       )

									snake_case__          =						((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths       ) - 1))
									snake_case__          =						int(config.embed_dim * 2 ** (len(config.depths       ) - 1)       )

									self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, expected_seq_len, expected_dim)       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						TFSwinForMaskedImageModeling(config=__UpperCamelCase       )
									snake_case__          =						model(__UpperCamelCase       )
									self.parent.assertEqual(
									    result.logits.shape					,   (self.batch_size, self.num_channels, self.image_size, self.image_size)       )

									# test greyscale images
									snake_case__          =						1
									snake_case__          =						TFSwinForMaskedImageModeling(__UpperCamelCase       )

									snake_case__          =						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]       )
									snake_case__          =						model(__UpperCamelCase       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, 1, self.image_size, self.image_size)       )






						def       __lowerCAmelCase(   self						:						Dict					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						int       ):



									'''simple docstring'''






									snake_case__          =						self.type_sequence_label_size
									snake_case__          =						TFSwinForImageClassification(__UpperCamelCase       )
									snake_case__          =						model(__UpperCamelCase					,   labels=__UpperCamelCase       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.type_sequence_label_size)       )

									# test greyscale images
									snake_case__          =						1
									snake_case__          =						TFSwinForImageClassification(__UpperCamelCase       )
									snake_case__          =						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]       )
									snake_case__          =						model(__UpperCamelCase       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.type_sequence_label_size)       )







						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						self.prepare_config_and_inputs()
									snake_case__    ,			snake_case__    ,			snake_case__          =						config_and_inputs
									snake_case__          =						{"""pixel_values""": pixel_values}
									return config, inputs_dict







@require_tf
class __magic_name__(						__lowerCAmelCase    ,		__lowerCAmelCase    ,		unittest.TestCase      ):
						UpperCAmelCase_		:					int    =  (
						    (
						        TFSwinModel,
						        TFSwinForImageClassification,
						        TFSwinForMaskedImageModeling,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase_		:					Dict    =  (
						    {"""feature-extraction""": TFSwinModel, """image-classification""": TFSwinForImageClassification}
						    if is_tf_available()
						    else {}
						)

						UpperCAmelCase_		:					Tuple    =  False
						UpperCAmelCase_		:					Union[str, Any]    =  False
						UpperCAmelCase_		:					Dict    =  False
						UpperCAmelCase_		:					List[Any]    =  False






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						TFSwinModelTester(self       )
									snake_case__          =						ConfigTester(self					,   config_class=__UpperCamelCase					,   embed_dim=3_7       )






						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									self.create_and_test_config_common_properties()
									self.config_tester.create_and_test_config_to_json_string()
									self.config_tester.create_and_test_config_to_json_file()
									self.config_tester.create_and_test_config_from_and_save_pretrained()
									self.config_tester.create_and_test_config_with_num_labels()
									self.config_tester.check_config_can_be_init_without_params()
									self.config_tester.check_config_arguments_init()






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_image_modeling(*__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_image_classification(*__UpperCamelCase       )






						@unittest.skip(reason="""Swin does not use inputs_embeds"""       )
						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									pass






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__          =						self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
												snake_case__          =						model_class(__UpperCamelCase       )
												self.assertIsInstance(model.get_input_embeddings()					,   tf.keras.layers.Layer       )
												snake_case__          =						model.get_output_embeddings()
												self.assertTrue(x is None or isinstance(__UpperCamelCase					,   tf.keras.layers.Dense       )       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__          =						self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
												snake_case__          =						model_class(__UpperCamelCase       )
												snake_case__          =						inspect.signature(model.call       )
												# signature.parameters is an OrderedDict => so arg_names order is deterministic
												snake_case__          =						[*signature.parameters.keys()]

												snake_case__          =						["""pixel_values"""]
												self.assertListEqual(arg_names[:1]					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Tuple       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__          =						self.model_tester.prepare_config_and_inputs_for_common()
									snake_case__          =						True

									for model_class in self.all_model_classes:
												snake_case__          =						True
												snake_case__          =						False
												snake_case__          =						True
												snake_case__          =						model_class(__UpperCamelCase       )
												snake_case__          =						model(**self._prepare_for_class(__UpperCamelCase					,   __UpperCamelCase       )       )
												snake_case__          =						outputs.attentions
												snake_case__          =						len(self.model_tester.depths       )
												self.assertEqual(len(__UpperCamelCase       )					,   __UpperCamelCase       )

												# check that output_attentions also work using config
												del inputs_dict["output_attentions"]
												snake_case__          =						True
												snake_case__          =						config.window_size**2
												snake_case__          =						model_class(__UpperCamelCase       )
												snake_case__          =						model(**self._prepare_for_class(__UpperCamelCase					,   __UpperCamelCase       )       )
												snake_case__          =						outputs.attentions
												self.assertEqual(len(__UpperCamelCase       )					,   __UpperCamelCase       )

												self.assertListEqual(
												    list(attentions[0].shape[-3:]       )					,   [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,   )
												snake_case__          =						len(__UpperCamelCase       )

												# Check attention is always last and order is fine
												snake_case__          =						True
												snake_case__          =						True
												snake_case__          =						model_class(__UpperCamelCase       )
												snake_case__          =						model(**self._prepare_for_class(__UpperCamelCase					,   __UpperCamelCase       )       )

												if hasattr(self.model_tester					,   """num_hidden_states_types"""       ):
															snake_case__          =						self.model_tester.num_hidden_states_types
												else:
															# also another +1 for reshaped_hidden_states
															snake_case__          =						2
												self.assertEqual(out_len + added_hidden_states					,   len(__UpperCamelCase       )       )

												snake_case__          =						outputs.attentions

												self.assertEqual(len(__UpperCamelCase       )					,   __UpperCamelCase       )

												self.assertListEqual(
												    list(self_attentions[0].shape[-3:]       )					,   [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,   )






						def       __lowerCAmelCase(   self						:						List[str]					,   __UpperCamelCase						:						Tuple					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						model_class(__UpperCamelCase       )
									snake_case__          =						model(**self._prepare_for_class(__UpperCamelCase					,   __UpperCamelCase       )       )
									snake_case__          =						outputs.hidden_states

									snake_case__          =						getattr(
									    self.model_tester					,   """expected_num_hidden_layers"""					,   len(self.model_tester.depths       ) + 1       )
									self.assertEqual(len(__UpperCamelCase       )					,   __UpperCamelCase       )

									# Swin has a different seq_length
									snake_case__          =						to_atuple(config.patch_size       )

									snake_case__          =						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

									self.assertListEqual(
									    list(hidden_states[0].shape[-2:]       )					,   [num_patches, self.model_tester.embed_dim]					,   )

									snake_case__          =						outputs.reshaped_hidden_states
									self.assertEqual(len(__UpperCamelCase       )					,   __UpperCamelCase       )

									snake_case__    ,			snake_case__    ,			snake_case__    ,			snake_case__          =						reshaped_hidden_states[0].shape

									snake_case__          =						tf.reshape(reshaped_hidden_states[0]					,   (batch_size, num_channels, height * width)       )
									snake_case__          =						tf.transpose(__UpperCamelCase					,   (0, 2, 1)       )

									self.assertListEqual(
									    list(reshaped_hidden_states.shape[-2:]       )					,   [num_patches, self.model_tester.embed_dim]					,   )






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__          =						self.model_tester.prepare_config_and_inputs_for_common()

									snake_case__          =						to_atuple(self.model_tester.image_size       )

									for model_class in self.all_model_classes:
												snake_case__          =						True
												self.check_hidden_states_output(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )

												# check that output_hidden_states also work using config
												del inputs_dict["output_hidden_states"]
												snake_case__          =						True

												self.check_hidden_states_output(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__          =						self.model_tester.prepare_config_and_inputs_for_common()
									snake_case__          =						3

									snake_case__          =						to_atuple(self.model_tester.image_size       )
									snake_case__          =						to_atuple(config.patch_size       )

									snake_case__          =						image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
									snake_case__          =						image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

									for model_class in self.all_model_classes:
												snake_case__          =						True
												self.check_hidden_states_output(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   (padded_height, padded_width)       )

												# check that output_hidden_states also work using config
												del inputs_dict["output_hidden_states"]
												snake_case__          =						True
												self.check_hidden_states_output(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   (padded_height, padded_width)       )







						@slow
						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												snake_case__          =						TFSwinModel.from_pretrained(__UpperCamelCase       )
												self.assertIsNotNone(__UpperCamelCase       )







@require_vision
@require_tf
class __magic_name__(						unittest.TestCase      ):






						@cached_property
						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									return (
									    AutoImageProcessor.from_pretrained("""microsoft/swin-tiny-patch4-window7-224"""       )
									    if is_vision_available()
									    else None
									)







						@slow
						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						TFSwinForImageClassification.from_pretrained("""microsoft/swin-tiny-patch4-window7-224"""       )
									snake_case__          =						self.default_image_processor

									snake_case__          =						Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""       )
									snake_case__          =						image_processor(images=__UpperCamelCase					,   return_tensors="""tf"""       )

									# forward pass
									snake_case__          =						model(__UpperCamelCase       )

									# verify the logits
									snake_case__          =						tf.TensorShape((1, 1_0_0_0)       )
									self.assertEqual(outputs.logits.shape					,   __UpperCamelCase       )
									snake_case__          =						tf.constant([-0.09_48, -0.64_54, -0.09_21]       )
									self.assertTrue(np.allclose(outputs.logits[0, :3]					,   __UpperCamelCase					,   atol=1E-4       )       )






import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						UpperCamelCase__					( lowerCAmelCase_				, unittest.TestCase  ):



     '''simple docstring'''

     __snake_case       :      Dict						      =     PreTrainedTokenizerFast
     __snake_case       :      Optional[int]						      =     False
     __snake_case       :      List[Any]						      =     True
     __snake_case       :      str						      =     "tokenizer_file"






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Union[str, Any]			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		False  # because we don't have pretrained_vocab_files_map
           super().setUp()
           SCREAMING_SNAKE_CASE						=		True

           SCREAMING_SNAKE_CASE						=		["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
           SCREAMING_SNAKE_CASE						=		"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

           # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
           SCREAMING_SNAKE_CASE						=		[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
           tokenizer.save_pretrained(self.tmpdirname			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Tuple:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Union[str, Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						List[Any]:




           '''simple docstring'''


           pass






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                       try:
                             SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
                             SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,**lowerCamelCase__			)

                             tokenizer.save_pretrained(self.tmpdirname			)
                             super().test_training_new_tokenizer_with_special_tokens_change()
                       finally:
                             # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                             # is restored
                             shutil.rmtree(self.tmpdirname			)
                             SCREAMING_SNAKE_CASE						=		tmpdirname_orig






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

           SCREAMING_SNAKE_CASE						=		("""a""" for _ in range(1000			))
           SCREAMING_SNAKE_CASE						=		tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase__		,length=1000		,vocab_size=50			)

           SCREAMING_SNAKE_CASE						=		new_tokenizer.encode("""a🤗"""			)
           self.assertEqual(lowerCamelCase__		,[64, 172, 253, 97, 245]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained("""bert-base-cased"""			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           SCREAMING_SNAKE_CASE						=		len(lowerCamelCase__			)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 json.dump(lowerCamelCase__		,open(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,"""w"""			)			)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			) + 1			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(lowerCamelCase__		,"""tokenizer.4.0.0.json"""			)		,os.path.join(lowerCamelCase__		,"""tokenizer.42.0.0.json"""			)			)
                 SCREAMING_SNAKE_CASE						=		["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(lowerCamelCase__			)
                 SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
                 self.assertEqual(len(lowerCamelCase__			)		,len(lowerCamelCase__			)			)
                 SCREAMING_SNAKE_CASE						=		json.loads(new_tokenizer._tokenizer.to_str()			)
                 self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		"""hf-internal-testing/test-two-tokenizers"""

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           SCREAMING_SNAKE_CASE						=		AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28997			)
           SCREAMING_SNAKE_CASE						=		json.loads(tokenizer._tokenizer.to_str()			)
           self.assertIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

           # Testing an older version by monkey-patching the version in the module it's used.
           import transformers as old_transformers

           SCREAMING_SNAKE_CASE						=		"""3.0.0"""
           SCREAMING_SNAKE_CASE						=		old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase__			)
           self.assertEqual(len(lowerCamelCase__			)		,28996			)
           SCREAMING_SNAKE_CASE						=		json.loads(old_tokenizer._tokenizer.to_str()			)
           self.assertNotIn("""huggingface"""		,json_tokenizer["""model"""]["""vocab"""]			)

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""			)
           SCREAMING_SNAKE_CASE						=		"""The Matrix is a 1999 science fiction action film."""

           with concurrent.futures.ThreadPoolExecutor() as executor:
                 SCREAMING_SNAKE_CASE						=		[executor.submit(self.fetch		,lowerCamelCase__		,lowerCamelCase__			) for i in range(10			)]
                 SCREAMING_SNAKE_CASE						=		[future.result() for future in futures]
                 self.assertEqual(lowerCamelCase__		,[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[int]		,lowerCamelCase__		:       Optional[int]		,lowerCamelCase__		:       str			)	->						Union[str, Any]:




           '''simple docstring'''


           return tokenizer.encode(lowerCamelCase__		,truncation="""longest_first"""		,padding="""longest"""			)


