import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling





def 				UpperCAmelCase						(    lowercase							):







				"""simple docstring"""



				__lowercase 			=							[2, 2, 6, 2] if '''tiny''' in model_name else [2, 2, 18, 2]
				__lowercase 			=							True if '''large''' in model_name or '''huge''' in model_name else False
				__lowercase 			=							True if '''large''' in model_name or '''huge''' in model_name else False
				__lowercase 			=							True if '''large''' in model_name or '''huge''' in model_name else False

				if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
								if "fl3" in model_name:
												__lowercase 			=							[3, 3, 3, 3]
												__lowercase 			=							[5, 5, 5, 5]
								elif "fl4" in model_name:
												__lowercase 			=							[4, 4, 4, 4]
												__lowercase 			=							[3, 3, 3, 3]

				if "tiny" in model_name or "small" in model_name or "base" in model_name:
								__lowercase 			=							[3, 3, 3, 3]
								if "lrf" in model_name:
												__lowercase 			=							[3, 3, 3, 3]
								else:
												__lowercase 			=							[2, 2, 2, 2]

				if "tiny" in model_name:
								__lowercase 			=							96
				elif "small" in model_name:
								__lowercase 			=							96
				elif "base" in model_name:
								__lowercase 			=							128
				elif "large" in model_name:
								__lowercase 			=							192
				elif "xlarge" in model_name:
								__lowercase 			=							256
				elif "huge" in model_name:
								__lowercase 			=							352

				# set label information
				__lowercase 			=							'''huggingface/label-files'''
				if "large" in model_name or "huge" in model_name:
								__lowercase 			=							'''imagenet-22k-id2label.json'''
				else:
								__lowercase 			=							'''imagenet-1k-id2label.json'''

				__lowercase 			=							json.load(open(hf_hub_download(lowercase							,			lowercase							,			repo_type='''dataset'''							)							,			'''r'''							)							)
				__lowercase 			=							{int(lowercase							): v for k, v in idalabel.items()}
				__lowercase 			=							{v: k for k, v in idalabel.items()}

				__lowercase 			=							FocalNetConfig(
				    embed_dim=lowercase							,			depths=lowercase							,			focal_levels=lowercase							,			focal_windows=lowercase							,			use_conv_embed=lowercase							,			idalabel=lowercase							,			labelaid=lowercase							,			use_post_layernorm=lowercase							,			use_layerscale=lowercase							,			)

				return config





def 				UpperCAmelCase						(    lowercase							):







				"""simple docstring"""



				if "patch_embed.proj" in name:
								__lowercase 			=							name.replace('''patch_embed.proj'''							,			'''embeddings.patch_embeddings.projection'''							)
				if "patch_embed.norm" in name:
								__lowercase 			=							name.replace('''patch_embed.norm'''							,			'''embeddings.norm'''							)
				if "layers" in name:
								__lowercase 			=							'''encoder.''' + name
				if "encoder.layers" in name:
								__lowercase 			=							name.replace('''encoder.layers'''							,			'''encoder.stages'''							)
				if "downsample.proj" in name:
								__lowercase 			=							name.replace('''downsample.proj'''							,			'''downsample.projection'''							)
				if "blocks" in name:
								__lowercase 			=							name.replace('''blocks'''							,			'''layers'''							)
				if "modulation.f.weight" in name or "modulation.f.bias" in name:
								__lowercase 			=							name.replace('''modulation.f'''							,			'''modulation.projection_in'''							)
				if "modulation.h.weight" in name or "modulation.h.bias" in name:
								__lowercase 			=							name.replace('''modulation.h'''							,			'''modulation.projection_context'''							)
				if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
								__lowercase 			=							name.replace('''modulation.proj'''							,			'''modulation.projection_out'''							)

				if name == "norm.weight":
								__lowercase 			=							'''layernorm.weight'''
				if name == "norm.bias":
								__lowercase 			=							'''layernorm.bias'''

				if "head" in name:
								__lowercase 			=							name.replace('''head'''							,			'''classifier'''							)
				else:
								__lowercase 			=							'''focalnet.''' + name

				return name





def 				UpperCAmelCase						(    lowercase							,			lowercase							,			lowercase=False							):







				"""simple docstring"""



				__lowercase 			=							{
				    '''focalnet-tiny''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth''',
				    '''focalnet-tiny-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth''',
				    '''focalnet-small''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth''',
				    '''focalnet-small-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth''',
				    '''focalnet-base''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth''',
				    '''focalnet-base-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth''',
				    '''focalnet-large-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth''',
				    '''focalnet-large-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth''',
				    '''focalnet-xlarge-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth''',
				    '''focalnet-xlarge-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth''',
				}
				# fmt: on

				__lowercase 			=							model_name_to_url[model_name]
				print('''Checkpoint URL: '''							,			lowercase							)
				__lowercase 			=							torch.hub.load_state_dict_from_url(lowercase							,			map_location='''cpu'''							)['''model''']

				# rename keys
				for key in state_dict.copy().keys():
								__lowercase 			=							state_dict.pop(lowercase							)
								__lowercase 			=							val

				__lowercase 			=							get_focalnet_config(lowercase							)
				__lowercase 			=							FocalNetForImageClassification(lowercase							)
				model.eval()

				# load state dict
				model.load_state_dict(lowercase							)

				# verify conversion
				__lowercase 			=							'''http://images.cocodataset.org/val2017/000000039769.jpg'''

				__lowercase 			=							BitImageProcessor(
				    do_resize=lowercase							,			size={'''shortest_edge''': 256}							,			resample=PILImageResampling.BILINEAR							,			do_center_crop=lowercase							,			crop_size=224							,			do_normalize=lowercase							,			image_mean=lowercase							,			image_std=lowercase							,			)
				__lowercase 			=							Image.open(requests.get(lowercase							,			stream=lowercase							).raw							)
				__lowercase 			=							processor(images=lowercase							,			return_tensors='''pt'''							)

				__lowercase 			=							transforms.Compose(
				    [
				        transforms.Resize(256							),
				        transforms.CenterCrop(224							),
				        transforms.ToTensor(),
				        transforms.Normalize(mean=[0.485, 0.456, 0.406]							,			std=[0.229, 0.224, 0.225]							),
				    ]							)

				__lowercase 			=							image_transforms(lowercase							).unsqueeze(0							)

				# verify pixel_values
				assert torch.allclose(inputs.pixel_values							,			lowercase							,			atol=1E-4							)

				__lowercase 			=							model(**lowercase							)

				__lowercase 			=							outputs.logits.argmax(-1							).item()
				print('''Predicted class:'''							,			model.config.idalabel[predicted_class_idx]							)

				print('''First values of logits:'''							,			outputs.logits[0, :3]							)

				if model_name == "focalnet-tiny":
								__lowercase 			=							torch.tensor([0.2166, -0.4368, 0.2191]							)
				elif model_name == "focalnet-tiny-lrf":
								__lowercase 			=							torch.tensor([1.1669, 0.0125, -0.1695]							)
				elif model_name == "focalnet-small":
								__lowercase 			=							torch.tensor([0.4917, -0.0430, 0.1341]							)
				elif model_name == "focalnet-small-lrf":
								__lowercase 			=							torch.tensor([-0.2588, -0.5342, -0.2331]							)
				elif model_name == "focalnet-base":
								__lowercase 			=							torch.tensor([-0.1655, -0.4090, -0.1730]							)
				elif model_name == "focalnet-base-lrf":
								__lowercase 			=							torch.tensor([0.5306, -0.0483, -0.3928]							)
				assert torch.allclose(outputs.logits[0, :3]							,			lowercase							,			atol=1E-4							)
				print('''Looks ok!'''							)

				if pytorch_dump_folder_path is not None:
								print(F"Saving model and processor of {model_name} to {pytorch_dump_folder_path}"							)
								model.save_pretrained(lowercase							)
								processor.save_pretrained(lowercase							)

				if push_to_hub:
								print(F"Pushing model and processor of {model_name} to the hub..."							)
								model.push_to_hub(F"{model_name}"							)
								processor.push_to_hub(F"{model_name}"							)


if __name__ == "__main__":
		__a  :     str       =  argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    """--model_name""",
		    default="""focalnet-tiny""",
		    type=str,
		    help="""Name of the FocalNet model you'd like to convert.""",
		)
		parser.add_argument(
		    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
		)
		parser.add_argument(
		    """--push_to_hub""",
		    action="""store_true""",
		    help="""Whether to push the model and processor to the hub.""",
		)

		__a  :     int       =  parser.parse_args()
		convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		__a  :     Tuple       =  """pt"""
elif is_tf_available():
		__a  :     int       =  """tf"""
else:
		__a  :     Tuple       =  """jax"""




class    _UpperCamelCase      (						_UpperCAmelCase						,unittest.TestCase ):


			"""simple docstring"""






			__a  : List[Any]			 =						ByTaTokenizer
			__a  : str			 =						False





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					List[Any]:

							'''simple docstring'''


							super().setUp()
							__lowercase 			=							ByTaTokenizer()
							tokenizer.save_pretrained(self.tmpdirname						)





			@cached_property
			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Any:

							'''simple docstring'''


							return ByTaTokenizer.from_pretrained('''google/byt5-small'''						)





			def        _SCREAMING_SNAKE_CASE			(   self      ,   **lowerCAmelCase__						)     ->					ByTaTokenizer:

							'''simple docstring'''


							return self.tokenizer_class.from_pretrained(self.tmpdirname      ,   **lowerCAmelCase__						)





			def        _SCREAMING_SNAKE_CASE			(   self      ,   lowerCAmelCase__      ,   lowerCAmelCase__=False      ,   lowerCAmelCase__=20      ,   lowerCAmelCase__=5						)     ->					Tuple[str, list]:

							'''simple docstring'''


							__lowercase 			=							[]
							for i in range(len(lowerCAmelCase__						)						):
											try:
															__lowercase 			=							tokenizer.decode([i]      ,   clean_up_tokenization_spaces=lowerCAmelCase__						)
											except UnicodeDecodeError:
															pass
											toks.append((i, tok)						)

							__lowercase 			=							list(filter(lambda lowerCAmelCase__						: re.match(R'''^[ a-zA-Z]+$'''      ,   t[1]						)      ,   lowerCAmelCase__						)						)
							__lowercase 			=							list(filter(lambda lowerCAmelCase__						: [t[0]] == tokenizer.encode(t[1]      ,   add_special_tokens=lowerCAmelCase__						)      ,   lowerCAmelCase__						)						)
							if max_length is not None and len(lowerCAmelCase__						) > max_length:
											__lowercase 			=							toks[:max_length]
							if min_length is not None and len(lowerCAmelCase__						) < min_length and len(lowerCAmelCase__						) > 0:
											while len(lowerCAmelCase__						) < min_length:
															__lowercase 			=							toks + toks
        # toks_str = [t[1] for t in toks]
							__lowercase 			=							[t[0] for t in toks]

							# Ensure consistency
							__lowercase 			=							tokenizer.decode(lowerCAmelCase__      ,   clean_up_tokenization_spaces=lowerCAmelCase__						)
							if " " not in output_txt and len(lowerCAmelCase__						) > 1:
											__lowercase 			=							(
											    tokenizer.decode([toks_ids[0]]      ,   clean_up_tokenization_spaces=lowerCAmelCase__						)
											    + ''' '''
											    + tokenizer.decode(toks_ids[1:]      ,   clean_up_tokenization_spaces=lowerCAmelCase__						)
											)
							if with_prefix_space:
											__lowercase 			=							''' ''' + output_txt
							__lowercase 			=							tokenizer.encode(lowerCAmelCase__      ,   add_special_tokens=lowerCAmelCase__						)
							return output_txt, output_ids





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Tuple:

							'''simple docstring'''


							__lowercase 			=							self.ta_base_tokenizer
							__lowercase 			=							tokenizer(['''hi</s>''', '''I went to the gym</s>''', '''</s>''']						)
							__lowercase 			=							tokenizer(['''hi''', '''I went to the gym''', '''''']						)
							self.assertListEqual(batch_with_eos_added['''input_ids''']      ,   batch_without_eos_added['''input_ids''']						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Dict:

							'''simple docstring'''


							__lowercase 			=							self.ta_base_tokenizer
							__lowercase 			=							'''Unicode €.'''
							__lowercase 			=							tokenizer(lowerCAmelCase__						)
							__lowercase 			=							[88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
							self.assertEqual(encoded['''input_ids''']      ,   lowerCAmelCase__						)

							# decoding
							__lowercase 			=							tokenizer.decode(lowerCAmelCase__						)
							self.assertEqual(lowerCAmelCase__      ,   '''Unicode €.</s>'''						)

							__lowercase 			=							tokenizer('''e è é ê ë'''						)
							__lowercase 			=							[1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
							self.assertEqual(encoded['''input_ids''']      ,   lowerCAmelCase__						)
							# decoding
							__lowercase 			=							tokenizer.decode(lowerCAmelCase__						)
							self.assertEqual(lowerCAmelCase__      ,   '''e è é ê ë</s>'''						)

							# encode/decode, but with `encode` instead of `__call__`
							self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''						)						)      ,   '''e è é ê ë</s>'''						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					List[Any]:

							'''simple docstring'''


							__lowercase 			=							self.ta_base_tokenizer
							__lowercase 			=							['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							# fmt: off
							__lowercase 			=							[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
							# fmt: on
							__lowercase 			=							tokenizer(lowerCAmelCase__      ,   padding=lowerCAmelCase__      ,   return_tensors=lowerCAmelCase__						)
							self.assertIsInstance(lowerCAmelCase__      ,   lowerCAmelCase__						)

							if FRAMEWORK != "jax":
											__lowercase 			=							list(batch.input_ids.numpy()[0]						)
							else:
											__lowercase 			=							list(batch.input_ids.tolist()[0]						)

							self.assertListEqual(lowerCAmelCase__      ,   lowerCAmelCase__						)

							self.assertEqual((2, 37)      ,   batch.input_ids.shape						)
							self.assertEqual((2, 37)      ,   batch.attention_mask.shape						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					int:

							'''simple docstring'''


							__lowercase 			=							self.ta_base_tokenizer
							__lowercase 			=							['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							__lowercase 			=							tokenizer(lowerCAmelCase__      ,   padding=lowerCAmelCase__      ,   return_tensors=lowerCAmelCase__						)
							# check if input_ids are returned and no decoder_input_ids
							self.assertIn('''input_ids'''      ,   lowerCAmelCase__						)
							self.assertIn('''attention_mask'''      ,   lowerCAmelCase__						)
							self.assertNotIn('''decoder_input_ids'''      ,   lowerCAmelCase__						)
							self.assertNotIn('''decoder_attention_mask'''      ,   lowerCAmelCase__						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Tuple:

							'''simple docstring'''


							__lowercase 			=							self.ta_base_tokenizer
							__lowercase 			=							[
							    '''Summary of the text.''',
							    '''Another summary.''',
							]
							__lowercase 			=							tokenizer(
							    text_target=lowerCAmelCase__      ,   max_length=32      ,   padding='''max_length'''      ,   truncation=lowerCAmelCase__      ,   return_tensors=lowerCAmelCase__						)
							self.assertEqual(32      ,   targets['''input_ids'''].shape[1]						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					int:

							'''simple docstring'''


							__lowercase 			=							self.ta_base_tokenizer
							__lowercase 			=							['''A long paragraph for summarization. </s>''']
							__lowercase 			=							['''Summary of the text. </s>''']
							# fmt: off
							__lowercase 			=							[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
							__lowercase 			=							[86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
							# fmt: on

							__lowercase 			=							tokenizer(lowerCAmelCase__      ,   text_target=lowerCAmelCase__						)

							self.assertEqual(lowerCAmelCase__      ,   batch['''input_ids'''][0]						)
							self.assertEqual(lowerCAmelCase__      ,   batch['''labels'''][0]						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					List[str]:

							'''simple docstring'''


							__lowercase 			=							self.get_tokenizers()
							for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"						):
															self.assertNotEqual(tokenizer.model_max_length      ,   42						)

        # Now let's start the test
							__lowercase 			=							self.get_tokenizers()
							for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"						):
															# Isolate this from the other tests because we save additional tokens/etc
															__lowercase 			=							tempfile.mkdtemp()

															__lowercase 			=							''' He is very happy, UNwant\u00E9d,running'''
															__lowercase 			=							tokenizer.encode(lowerCAmelCase__      ,   add_special_tokens=lowerCAmelCase__						)
															tokenizer.save_pretrained(lowerCAmelCase__						)

															__lowercase 			=							tokenizer.__class__.from_pretrained(lowerCAmelCase__						)
															__lowercase 			=							after_tokenizer.encode(lowerCAmelCase__      ,   add_special_tokens=lowerCAmelCase__						)
															self.assertListEqual(lowerCAmelCase__      ,   lowerCAmelCase__						)

															shutil.rmtree(lowerCAmelCase__						)

							__lowercase 			=							self.get_tokenizers(model_max_length=42						)
							for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"						):
															# Isolate this from the other tests because we save additional tokens/etc
															__lowercase 			=							tempfile.mkdtemp()

															__lowercase 			=							''' He is very happy, UNwant\u00E9d,running'''
															tokenizer.add_tokens(['''bim''', '''bambam''']						)
															__lowercase 			=							tokenizer.additional_special_tokens
															additional_special_tokens.append('''new_additional_special_token'''						)
															tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}						)
															__lowercase 			=							tokenizer.encode(lowerCAmelCase__      ,   add_special_tokens=lowerCAmelCase__						)
															tokenizer.save_pretrained(lowerCAmelCase__						)

															__lowercase 			=							tokenizer.__class__.from_pretrained(lowerCAmelCase__						)
															__lowercase 			=							after_tokenizer.encode(lowerCAmelCase__      ,   add_special_tokens=lowerCAmelCase__						)
															self.assertListEqual(lowerCAmelCase__      ,   lowerCAmelCase__						)
															self.assertIn('''new_additional_special_token'''      ,   after_tokenizer.additional_special_tokens						)
															self.assertEqual(after_tokenizer.model_max_length      ,   42						)

															__lowercase 			=							tokenizer.__class__.from_pretrained(lowerCAmelCase__      ,   model_max_length=43						)
															self.assertEqual(tokenizer.model_max_length      ,   43						)

															shutil.rmtree(lowerCAmelCase__						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Dict:

							'''simple docstring'''


							__lowercase 			=							[]
							if self.test_slow_tokenizer:
											tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

							if self.test_rust_tokenizer:
											tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
											with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer_utils.save_pretrained(lowerCAmelCase__						)

															with open(os.path.join(lowerCAmelCase__      ,   '''special_tokens_map.json'''						)      ,   encoding='''utf-8'''						) as json_file:
																			__lowercase 			=							json.load(lowerCAmelCase__						)

															with open(os.path.join(lowerCAmelCase__      ,   '''tokenizer_config.json'''						)      ,   encoding='''utf-8'''						) as json_file:
																			__lowercase 			=							json.load(lowerCAmelCase__						)

															__lowercase 			=							[F"<extra_id_{i}>" for i in range(1_25						)]

															__lowercase 			=							added_tokens_extra_ids + [
															    '''an_additional_special_token'''
															]
															__lowercase 			=							added_tokens_extra_ids + [
															    '''an_additional_special_token'''
															]

															with open(os.path.join(lowerCAmelCase__      ,   '''special_tokens_map.json'''						)      ,   '''w'''      ,   encoding='''utf-8'''						) as outfile:
																			json.dump(lowerCAmelCase__      ,   lowerCAmelCase__						)
															with open(os.path.join(lowerCAmelCase__      ,   '''tokenizer_config.json'''						)      ,   '''w'''      ,   encoding='''utf-8'''						) as outfile:
																			json.dump(lowerCAmelCase__      ,   lowerCAmelCase__						)

															# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
															# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
															# "special_tokens_map.json" files
															__lowercase 			=							tokenizer_class.from_pretrained(
															    lowerCAmelCase__      ,   )
															self.assertIn(
															    '''an_additional_special_token'''      ,   tokenizer_without_change_in_init.additional_special_tokens						)
															# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
															self.assertEqual(
															    ['''an_additional_special_token''']      ,   tokenizer_without_change_in_init.convert_ids_to_tokens(
															        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']						)						)      ,   )

															# Now we test that we can change the value of additional_special_tokens in the from_pretrained
															__lowercase 			=							added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''      ,   lstrip=lowerCAmelCase__						)]
															__lowercase 			=							tokenizer_class.from_pretrained(
															    lowerCAmelCase__      ,   additional_special_tokens=lowerCAmelCase__      ,   )

															self.assertIn('''a_new_additional_special_token'''      ,   tokenizer.additional_special_tokens						)
															self.assertEqual(
															    ['''a_new_additional_special_token''']      ,   tokenizer.convert_ids_to_tokens(
															        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']						)						)      ,   )





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					List[str]:

							'''simple docstring'''


							__lowercase 			=							[]
							if self.test_slow_tokenizer:
											tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

							if self.test_rust_tokenizer:
											tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
											with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer_utils.save_pretrained(lowerCAmelCase__						)

															__lowercase 			=							tokenizer_class.from_pretrained(lowerCAmelCase__						)

															self.assertTrue(tokenizer.decode([2_55]						) == ''''''						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Optional[int]:

							'''simple docstring'''


							pass





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Union[str, Any]:

							'''simple docstring'''


							pass





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					int:

							'''simple docstring'''


							pass





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					str:

							'''simple docstring'''


							pass





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Optional[int]:

							'''simple docstring'''


							__lowercase 			=							self.get_tokenizers(fast=lowerCAmelCase__      ,   do_lower_case=lowerCAmelCase__						)
							for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"						):
															__lowercase 			=							['''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''x''', '''t''', '''</s>''']
															__lowercase 			=							tokenizer.convert_tokens_to_string(lowerCAmelCase__						)

															self.assertIsInstance(lowerCAmelCase__      ,   lowerCAmelCase__						)





			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					int:

							'''simple docstring'''


							__lowercase 			=							self.get_tokenizers()
							for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"						):
															__lowercase 			=							[
															    '''bos_token''',
															    '''eos_token''',
															    '''unk_token''',
															    '''sep_token''',
															    '''pad_token''',
															    '''cls_token''',
															    '''mask_token''',
															]

															__lowercase 			=							0
															__lowercase 			=							tokenizer.convert_ids_to_tokens(
															    lowerCAmelCase__      ,   skip_special_tokens=lowerCAmelCase__						)

															for attr in attributes_list:
																			setattr(lowerCAmelCase__      ,   attr + '''_id'''      ,   lowerCAmelCase__						)
																			self.assertEqual(getattr(lowerCAmelCase__      ,   lowerCAmelCase__						)      ,   lowerCAmelCase__						)
																			self.assertEqual(getattr(lowerCAmelCase__      ,   attr + '''_id'''						)      ,   lowerCAmelCase__						)

																			setattr(lowerCAmelCase__      ,   attr + '''_id'''      ,   lowerCAmelCase__						)
																			self.assertEqual(getattr(lowerCAmelCase__      ,   lowerCAmelCase__						)      ,   lowerCAmelCase__						)
																			self.assertEqual(getattr(lowerCAmelCase__      ,   attr + '''_id'''						)      ,   lowerCAmelCase__						)

															setattr(lowerCAmelCase__      ,   '''additional_special_tokens_ids'''      ,   []						)
															self.assertListEqual(getattr(lowerCAmelCase__      ,   '''additional_special_tokens'''						)      ,   []						)
															self.assertListEqual(getattr(lowerCAmelCase__      ,   '''additional_special_tokens_ids'''						)      ,   []						)

															setattr(lowerCAmelCase__      ,   '''additional_special_tokens_ids'''      ,   [token_id_to_test_setters]						)
															self.assertListEqual(getattr(lowerCAmelCase__      ,   '''additional_special_tokens'''						)      ,   [token_to_test_setters]						)
															self.assertListEqual(getattr(lowerCAmelCase__      ,   '''additional_special_tokens_ids'''						)      ,   [token_id_to_test_setters]						)