code: 
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class 					snake_case__  ( __A      ,    unittest.TestCase       ):
  	UpperCAmelCase   :							Dict         = PreTrainedTokenizerFast
  	UpperCAmelCase   :							str         = False
  	UpperCAmelCase   :							Optional[Any]         = True
  	UpperCAmelCase   :							Tuple         = """tokenizer_file"""
  
  
  
  
  
  	def     A				(  self							)		->			Optional[Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   List[str]			=	False  # because we don't have pretrained_vocab_files_map
  								super().setUp()
  								a_       :   Tuple			=	True
  
  								a_       :   Union[str, Any]			=	["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
  								a_       :   Optional[Any]			=	"""SaulLu/dummy-tokenizer-bytelevel-bpe"""
  
  								# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
  								a_       :   Any			=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
  								a_       :   Dict			=	PreTrainedTokenizerFast.from_pretrained(model_paths[0]							)
  								tokenizer.save_pretrained(self.tmpdirname							)
  
  
  
  
  
  	def     A				(  self							)		->			List[Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								pass
  
  
  
  
  
  	def     A				(  self							)		->			Any:
  								"""simple docstring"""
  
  
  
  
  
  
  								pass
  
  
  
  
  
  	def     A				(  self							)		->			Dict:
  								"""simple docstring"""
  
  
  
  
  
  
  								pass
  
  
  
  
  
  	def     A				(  self							)		->			Optional[int]:
  								"""simple docstring"""
  
  
  
  
  
  
  								pass
  
  
  
  
  
  	def     A				(  self							)		->			List[Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Tuple			=	self.tmpdirname
  								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
  																						try:
  																													a_       :   List[Any]			=	tempfile.mkdtemp()
  																													a_       :   Optional[Any]			=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase_				,    **UpperCamelCase_							)
  
  																													tokenizer.save_pretrained(self.tmpdirname							)
  																													super().test_training_new_tokenizer()
  																						finally:
  																													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  																													# is restored
  																													shutil.rmtree(self.tmpdirname							)
  																													a_       :   Optional[Any]			=	tmpdirname_orig
  
  
  
  
  
  	def     A				(  self							)		->			Union[str, Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Optional[Any]			=	self.tmpdirname
  								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
  																						try:
  																													a_       :   Optional[Any]			=	tempfile.mkdtemp()
  																													a_       :   Any			=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase_				,    **UpperCamelCase_							)
  
  																													tokenizer.save_pretrained(self.tmpdirname							)
  																													super().test_training_new_tokenizer_with_special_tokens_change()
  																						finally:
  																													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  																													# is restored
  																													shutil.rmtree(self.tmpdirname							)
  																													a_       :   Tuple			=	tmpdirname_orig
  
  
  
  
  
  	def     A				(  self							)		->			Dict:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   str			=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name							)
  
  								a_       :   Union[str, Any]			=	("""a""" for _ in range(1000							))
  								a_       :   Union[str, Any]			=	tokenizer.train_new_from_iterator(text_iterator=UpperCamelCase_				,    length=1000				,    vocab_size=50							)
  
  								a_       :   Tuple			=	new_tokenizer.encode("""a🤗"""							)
  								self.assertEqual(UpperCamelCase_				,    [64, 172, 253, 97, 245]							)
  
  
  
  
  
  @require_tokenizers
  class 					snake_case__  ( unittest.TestCase       ):
  
  
  
  
  
  	def     A				(  self							)		->			Optional[int]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   int			=	AutoTokenizer.from_pretrained("""bert-base-cased"""							)
  								a_       :   Union[str, Any]			=	json.loads(tokenizer._tokenizer.to_str()							)
  								a_       :   Optional[Any]			=	len(UpperCamelCase_							)
  
  								with tempfile.TemporaryDirectory() as tmp_dir:
  															# Hack to save this in the tokenizer_config.json
  															a_       :   List[Any]			=	["""tokenizer.4.0.0.json"""]
  															tokenizer.save_pretrained(UpperCamelCase_							)
  															json.dump(UpperCamelCase_				,    open(os.path.join(UpperCamelCase_				,    """tokenizer.4.0.0.json"""							)				,    """w"""							)							)
  
  															# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  															a_       :   str			=	AutoTokenizer.from_pretrained(UpperCamelCase_							)
  															self.assertEqual(len(UpperCamelCase_							)				,    len(UpperCamelCase_							) + 1							)
  															a_       :   Tuple			=	json.loads(new_tokenizer._tokenizer.to_str()							)
  															self.assertIn("""huggingface"""				,    json_tokenizer["""model"""]["""vocab"""]							)
  
  															# Will need to be adjusted if we reach v42 and this test is still here.
  															# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
  															shutil.move(os.path.join(UpperCamelCase_				,    """tokenizer.4.0.0.json"""							)				,    os.path.join(UpperCamelCase_				,    """tokenizer.42.0.0.json"""							)							)
  															a_       :   Union[str, Any]			=	["""tokenizer.42.0.0.json"""]
  															tokenizer.save_pretrained(UpperCamelCase_							)
  															a_       :   List[Any]			=	AutoTokenizer.from_pretrained(UpperCamelCase_							)
  															self.assertEqual(len(UpperCamelCase_							)				,    len(UpperCamelCase_							)							)
  															a_       :   Dict			=	json.loads(new_tokenizer._tokenizer.to_str()							)
  															self.assertNotIn("""huggingface"""				,    json_tokenizer["""model"""]["""vocab"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   str			=	"""hf-internal-testing/test-two-tokenizers"""
  
  								# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  								a_       :   Any			=	AutoTokenizer.from_pretrained(UpperCamelCase_							)
  								self.assertEqual(len(UpperCamelCase_							)				,    28997							)
  								a_       :   int			=	json.loads(tokenizer._tokenizer.to_str()							)
  								self.assertIn("""huggingface"""				,    json_tokenizer["""model"""]["""vocab"""]							)
  
  								# Testing an older version by monkey-patching the version in the module it's used.
  								import transformers as old_transformers
  
  								a_       :   Any			=	"""3.0.0"""
  								a_       :   Dict			=	old_transformers.models.auto.AutoTokenizer.from_pretrained(UpperCamelCase_							)
  								self.assertEqual(len(UpperCamelCase_							)				,    28996							)
  								a_       :   List[str]			=	json.loads(old_tokenizer._tokenizer.to_str()							)
  								self.assertNotIn("""huggingface"""				,    json_tokenizer["""model"""]["""vocab"""]							)
  
  
  
  
  
  @require_tokenizers
  class 					snake_case__  ( unittest.TestCase       ):
  
  
  
  
  
  	def     A				(  self							)		->			List[str]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   List[Any]			=	PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""							)
  								a_       :   Optional[Any]			=	"""The Matrix is a 1999 science fiction action film."""
  
  								with concurrent.futures.ThreadPoolExecutor() as executor:
  															a_       :   Dict			=	[executor.submit(self.fetch				,    UpperCamelCase_				,    UpperCamelCase_							) for i in range(10							)]
  															a_       :   Optional[Any]			=	[future.result() for future in futures]
  															self.assertEqual(UpperCamelCase_				,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10							)]							)
  
  
  
  
  
  	def     A				(  self				,    UpperCamelCase_				,    UpperCamelCase_							)		->			List[Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								return tokenizer.encode(UpperCamelCase_				,    truncation="""longest_first"""				,    padding="""longest"""							)
  
  
  
style_context: 
  
  
  from collections import OrderedDict
  from typing import Mapping
  
  from packaging import version
  
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig
  from ...utils import logging
  
  
  SCREAMING_SNAKE_CASE : Dict          =    logging.get_logger(__name__)
  
  SCREAMING_SNAKE_CASE : str          =    {
      "apple/mobilevit-small": "https://huggingface.co/apple/mobilevit-small/resolve/main/config.json",
      "apple/mobilevit-x-small": "https://huggingface.co/apple/mobilevit-x-small/resolve/main/config.json",
      "apple/mobilevit-xx-small": "https://huggingface.co/apple/mobilevit-xx-small/resolve/main/config.json",
      "apple/deeplabv3-mobilevit-small": (
          "https://huggingface.co/apple/deeplabv3-mobilevit-small/resolve/main/config.json"
      ),
      "apple/deeplabv3-mobilevit-x-small": (
          "https://huggingface.co/apple/deeplabv3-mobilevit-x-small/resolve/main/config.json"
      ),
      "apple/deeplabv3-mobilevit-xx-small": (
          "https://huggingface.co/apple/deeplabv3-mobilevit-xx-small/resolve/main/config.json"
      ),
      # See all MobileViT models at https://huggingface.co/models?filter=mobilevit
  }
  
  
  
  
  
  class 					snake_case__  ( __A       ):
  	UpperCAmelCase   :							List[str]         = """mobilevit"""
  
  
  
  
  
  	def __init__(  self				,    UpperCamelCase_=3				,    UpperCamelCase_=256				,    UpperCamelCase_=2				,    UpperCamelCase_=[144, 192, 240]				,    UpperCamelCase_=[16, 32, 64, 96, 128, 160, 640]				,    UpperCamelCase_=4				,    UpperCamelCase_=2.0				,    UpperCamelCase_=4.0				,    UpperCamelCase_="silu"				,    UpperCamelCase_=3				,    UpperCamelCase_=32				,    UpperCamelCase_=0.1				,    UpperCamelCase_=0.0				,    UpperCamelCase_=0.1				,    UpperCamelCase_=0.02				,    UpperCamelCase_=1e-5				,    UpperCamelCase_=True				,    UpperCamelCase_=256				,    UpperCamelCase_=[6, 12, 18]				,    UpperCamelCase_=0.1				,    UpperCamelCase_=255				,    **UpperCamelCase_				,    )		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								super().__init__(**UpperCamelCase_							)
  
  								a_       :   Optional[int]			=	num_channels
  								a_       :   Optional[Any]			=	image_size
  								a_       :   int			=	patch_size
  								a_       :   List[str]			=	hidden_sizes
  								a_       :   Optional[int]			=	neck_hidden_sizes
  								a_       :   List[Any]			=	num_attention_heads
  								a_       :   Union[str, Any]			=	mlp_ratio
  								a_       :   List[str]			=	expand_ratio
  								a_       :   str			=	hidden_act
  								a_       :   Union[str, Any]			=	conv_kernel_size
  								a_       :   Optional[Any]			=	output_stride
  								a_       :   Union[str, Any]			=	hidden_dropout_prob
  								a_       :   List[Any]			=	attention_probs_dropout_prob
  								a_       :   List[Any]			=	classifier_dropout_prob
  								a_       :   Union[str, Any]			=	initializer_range
  								a_       :   List[Any]			=	layer_norm_eps
  								a_       :   str			=	qkv_bias
  
  								# decode head attributes for semantic segmentation
  								a_       :   Optional[Any]			=	aspp_out_channels
  								a_       :   Any			=	atrous_rates
  								a_       :   Union[str, Any]			=	aspp_dropout_prob
  								a_       :   Any			=	semantic_loss_ignore_index
  
  
  
  
  
  class 					snake_case__  ( __A       ):
  	UpperCAmelCase   :							Optional[int]         = version.parse("""1.11"""       )
  
  
  
  
  
  	@property
  	def     A				(  self							)		->			Mapping[str, Mapping[int, str]]:
  								"""simple docstring"""
  
  
  
  
  
  
  								return OrderedDict([("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""})]							)
  
  
  
  
  
  	@property
  	def     A				(  self							)		->			Mapping[str, Mapping[int, str]]:
  								"""simple docstring"""
  
  
  
  
  
  
  								if self.task == "image-classification":
  															return OrderedDict([("""logits""", {0: """batch"""})]							)
  								else:
  															return OrderedDict([("""last_hidden_state""", {0: """batch"""}), ("""pooler_output""", {0: """batch"""})]							)
  
  
  
  
  
  	@property
  	def     A				(  self							)		->			float:
  								"""simple docstring"""
  
  
  
  
  
  
  								return 1e-4
  
  
  