"""simple docstring"""







import argparse
import os.path as osp
import re

import torch
from safetensors.torch import load_file, save_file


# =================#
# UNet Conversion #
# =================#

_a     :     Optional[int]= [
    # (stable-diffusion, HF Diffusers)
    ("time_embed.0.weight", "time_embedding.linear_1.weight"),
    ("time_embed.0.bias", "time_embedding.linear_1.bias"),
    ("time_embed.2.weight", "time_embedding.linear_2.weight"),
    ("time_embed.2.bias", "time_embedding.linear_2.bias"),
    ("input_blocks.0.0.weight", "conv_in.weight"),
    ("input_blocks.0.0.bias", "conv_in.bias"),
    ("out.0.weight", "conv_norm_out.weight"),
    ("out.0.bias", "conv_norm_out.bias"),
    ("out.2.weight", "conv_out.weight"),
    ("out.2.bias", "conv_out.bias"),
]

_a     :     List[Any]= [
    # (stable-diffusion, HF Diffusers)
    ("in_layers.0", "norm1"),
    ("in_layers.2", "conv1"),
    ("out_layers.0", "norm2"),
    ("out_layers.3", "conv2"),
    ("emb_layers.1", "time_emb_proj"),
    ("skip_connection", "conv_shortcut"),
]

_a     :     Optional[Any]= []
# hardcoded number of downblocks and resnets/attentions...
# would need smarter logic for other networks.
for i in range(4):
	# loop over downblocks/upblocks

	for j in range(2):
		# loop over resnets/attentions for downblocks
		_a     :     str= f'''down_blocks.{i}.resnets.{j}.'''
		_a     :     Union[str, Any]= f'''input_blocks.{3*i + j + 1}.0.'''
		unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))

		if i < 3:
			# no attention layers in down_blocks.3
			_a     :     Tuple= f'''down_blocks.{i}.attentions.{j}.'''
			_a     :     Dict= f'''input_blocks.{3*i + j + 1}.1.'''
			unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))

	for j in range(3):
		# loop over resnets/attentions for upblocks
		_a     :     List[Any]= f'''up_blocks.{i}.resnets.{j}.'''
		_a     :     int= f'''output_blocks.{3*i + j}.0.'''
		unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))

		if i > 0:
			# no attention layers in up_blocks.0
			_a     :     List[str]= f'''up_blocks.{i}.attentions.{j}.'''
			_a     :     Tuple= f'''output_blocks.{3*i + j}.1.'''
			unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))

	if i < 3:
		# no downsample in down_blocks.3
		_a     :     List[str]= f'''down_blocks.{i}.downsamplers.0.conv.'''
		_a     :     Any= f'''input_blocks.{3*(i+1)}.0.op.'''
		unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))

		# no upsample in up_blocks.3
		_a     :     Optional[int]= f'''up_blocks.{i}.upsamplers.0.'''
		_a     :     int= f'''output_blocks.{3*i + 2}.{1 if i == 0 else 2}.'''
		unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))

_a     :     Union[str, Any]= "mid_block.attentions.0."
_a     :     List[Any]= "middle_block.1."
unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))

for j in range(2):
	_a     :     Tuple= f'''mid_block.resnets.{j}.'''
	_a     :     List[str]= f'''middle_block.{2*j}.'''
	unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
def   __UpperCAmelCase				(    UpperCAmelCase_    :				str  )							->			Union[str, Any]:

	'''simple docstring'''





	__snake_case			:							List[str]      		= {k: k for k in unet_state_dict.keys()}
	for sd_name, hf_name in unet_conversion_map:
		__snake_case			:							Tuple      		= sd_name
	for k, v in mapping.items():
		if "resnets" in k:
			for sd_part, hf_part in unet_conversion_map_resnet:
				__snake_case			:							Optional[int]      		= v.replace(_A			,							_A  )
			__snake_case			:							Optional[Any]      		= v
	for k, v in mapping.items():
		for sd_part, hf_part in unet_conversion_map_layer:
			__snake_case			:							Any      		= v.replace(_A			,							_A  )
		__snake_case			:							str      		= v
	__snake_case			:							Optional[Any]      		= {v: unet_state_dict[k] for k, v in mapping.items()}
	return new_state_dict


# ================#
# VAE Conversion #
# ================#

_a     :     Dict= [
    # (stable-diffusion, HF Diffusers)
    ("nin_shortcut", "conv_shortcut"),
    ("norm_out", "conv_norm_out"),
    ("mid.attn_1.", "mid_block.attentions.0."),
]

for i in range(4):
	# down_blocks have two resnets
	for j in range(2):
		_a     :     Any= f'''encoder.down_blocks.{i}.resnets.{j}.'''
		_a     :     List[Any]= f'''encoder.down.{i}.block.{j}.'''
		vae_conversion_map.append((sd_down_prefix, hf_down_prefix))

	if i < 3:
		_a     :     Optional[int]= f'''down_blocks.{i}.downsamplers.0.'''
		_a     :     Tuple= f'''down.{i}.downsample.'''
		vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix))

		_a     :     List[str]= f'''up_blocks.{i}.upsamplers.0.'''
		_a     :     Optional[int]= f'''up.{3-i}.upsample.'''
		vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix))

	# up_blocks have three resnets
	# also, up blocks in hf are numbered in reverse from sd
	for j in range(3):
		_a     :     int= f'''decoder.up_blocks.{i}.resnets.{j}.'''
		_a     :     Union[str, Any]= f'''decoder.up.{3-i}.block.{j}.'''
		vae_conversion_map.append((sd_up_prefix, hf_up_prefix))

# this part accounts for mid blocks in both the encoder and the decoder
for i in range(2):
	_a     :     Dict= f'''mid_block.resnets.{i}.'''
	_a     :     int= f'''mid.block_{i+1}.'''
	vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix))


_a     :     Optional[Any]= [
    # (stable-diffusion, HF Diffusers)
    ("norm.", "group_norm."),
    ("q.", "query."),
    ("k.", "key."),
    ("v.", "value."),
    ("proj_out.", "proj_attn."),
]
def   __UpperCAmelCase				(    UpperCAmelCase_    :				Dict  )							->			Union[str, Any]:

	'''simple docstring'''





	return w.reshape(*w.shape			,							1			,							1  )
def   __UpperCAmelCase				(    UpperCAmelCase_    :				List[Any]  )							->			List[Any]:

	'''simple docstring'''





	__snake_case			:							Tuple      		= {k: k for k in vae_state_dict.keys()}
	for k, v in mapping.items():
		for sd_part, hf_part in vae_conversion_map:
			__snake_case			:							Optional[int]      		= v.replace(_A			,							_A  )
		__snake_case			:							List[Any]      		= v
	for k, v in mapping.items():
		if "attentions" in k:
			for sd_part, hf_part in vae_conversion_map_attn:
				__snake_case			:							List[str]      		= v.replace(_A			,							_A  )
			__snake_case			:							Dict      		= v
	__snake_case			:							Dict      		= {v: vae_state_dict[k] for k, v in mapping.items()}
	__snake_case			:							Any      		= ['q', 'k', 'v', 'proj_out']
	for k, v in new_state_dict.items():
		for weight_name in weights_to_convert:
			if F"mid.attn_1.{weight_name}.weight" in k:
				print(F"Reshaping {k} for SD format"  )
				__snake_case			:							Optional[Any]      		= reshape_weight_for_sd(_A  )
	return new_state_dict


# =========================#
# Text Encoder Conversion #
# =========================#


_a     :     int= [
    # (stable-diffusion, HF Diffusers)
    ("resblocks.", "text_model.encoder.layers."),
    ("ln_1", "layer_norm1"),
    ("ln_2", "layer_norm2"),
    (".c_fc.", ".fc1."),
    (".c_proj.", ".fc2."),
    (".attn", ".self_attn"),
    ("ln_final.", "transformer.text_model.final_layer_norm."),
    ("token_embedding.weight", "transformer.text_model.embeddings.token_embedding.weight"),
    ("positional_embedding", "transformer.text_model.embeddings.position_embedding.weight"),
]
_a     :     Dict= {re.escape(x[1]): x[0] for x in textenc_conversion_lst}
_a     :     Tuple= re.compile("|".join(protected.keys()))

# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp
_a     :     Dict= {"q": 0, "k": 1, "v": 2}
def   __UpperCAmelCase				(    UpperCAmelCase_    :				Dict  )							->			int:

	'''simple docstring'''





	__snake_case			:							Any      		= {}
	__snake_case			:							Optional[int]      		= {}
	__snake_case			:							List[str]      		= {}
	for k, v in text_enc_dict.items():
		if (
		    k.endswith('.self_attn.q_proj.weight'  )
		    or k.endswith('.self_attn.k_proj.weight'  )
		    or k.endswith('.self_attn.v_proj.weight'  )
		):
			__snake_case			:							Optional[int]      		= k[: -len('.q_proj.weight'  )]
			__snake_case			:							List[Any]      		= k[-len('q_proj.weight'  )]
			if k_pre not in capture_qkv_weight:
				__snake_case			:							str      		= [None, None, None]
			__snake_case			:							int      		= v
			continue

		if (
		    k.endswith('.self_attn.q_proj.bias'  )
		    or k.endswith('.self_attn.k_proj.bias'  )
		    or k.endswith('.self_attn.v_proj.bias'  )
		):
			__snake_case			:							Optional[Any]      		= k[: -len('.q_proj.bias'  )]
			__snake_case			:							Optional[int]      		= k[-len('q_proj.bias'  )]
			if k_pre not in capture_qkv_bias:
				__snake_case			:							Union[str, Any]      		= [None, None, None]
			__snake_case			:							Optional[Any]      		= v
			continue

		__snake_case			:							str      		= textenc_pattern.sub(lambda UpperCAmelCase_  : protected[re.escape(m.group(0  )  )]			,							_A  )
		__snake_case			:							Optional[Any]      		= v

	for k_pre, tensors in capture_qkv_weight.items():
		if None in tensors:
			raise Exception('CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing'  )
		__snake_case			:							List[str]      		= textenc_pattern.sub(lambda UpperCAmelCase_  : protected[re.escape(m.group(0  )  )]			,							_A  )
		__snake_case			:							Optional[Any]      		= torch.cat(_A  )

	for k_pre, tensors in capture_qkv_bias.items():
		if None in tensors:
			raise Exception('CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing'  )
		__snake_case			:							Union[str, Any]      		= textenc_pattern.sub(lambda UpperCAmelCase_  : protected[re.escape(m.group(0  )  )]			,							_A  )
		__snake_case			:							List[Any]      		= torch.cat(_A  )

	return new_state_dict
def   __UpperCAmelCase				(    UpperCAmelCase_    :				Optional[Any]  )							->			List[Any]:

	'''simple docstring'''





	return text_enc_dict


if __name__ == "__main__":
	_a     :     int= argparse.ArgumentParser()

	parser.add_argument("--model_path", default=None, type=str, required=True, help="Path to the model to convert.")
	parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output model.")
	parser.add_argument("--half", action="store_true", help="Save weights in half precision.")
	parser.add_argument(
	    "--use_safetensors", action="store_true", help="Save weights use safetensors, default is ckpt."
	)

	_a     :     Dict= parser.parse_args()

	assert args.model_path is not None, "Must provide a model path!"

	assert args.checkpoint_path is not None, "Must provide a checkpoint path!"

	# Path for safetensors
	_a     :     Tuple= osp.join(args.model_path, "unet", "diffusion_pytorch_model.safetensors")
	_a     :     int= osp.join(args.model_path, "vae", "diffusion_pytorch_model.safetensors")
	_a     :     Any= osp.join(args.model_path, "text_encoder", "model.safetensors")

	# Load models from safetensors if it exists, if it doesn't pytorch
	if osp.exists(unet_path):
		_a     :     str= load_file(unet_path, device="cpu")
	else:
		_a     :     Optional[Any]= osp.join(args.model_path, "unet", "diffusion_pytorch_model.bin")
		_a     :     Any= torch.load(unet_path, map_location="cpu")

	if osp.exists(vae_path):
		_a     :     Tuple= load_file(vae_path, device="cpu")
	else:
		_a     :     Any= osp.join(args.model_path, "vae", "diffusion_pytorch_model.bin")
		_a     :     Dict= torch.load(vae_path, map_location="cpu")

	if osp.exists(text_enc_path):
		_a     :     Union[str, Any]= load_file(text_enc_path, device="cpu")
	else:
		_a     :     Union[str, Any]= osp.join(args.model_path, "text_encoder", "pytorch_model.bin")
		_a     :     Optional[int]= torch.load(text_enc_path, map_location="cpu")

	# Convert the UNet model
	_a     :     Dict= convert_unet_state_dict(unet_state_dict)
	_a     :     Any= {"model.diffusion_model." + k: v for k, v in unet_state_dict.items()}

	# Convert the VAE model
	_a     :     Dict= convert_vae_state_dict(vae_state_dict)
	_a     :     Union[str, Any]= {"first_stage_model." + k: v for k, v in vae_state_dict.items()}

	# Easiest way to identify v2.0 model seems to be that the text encoder (OpenCLIP) is deeper
	_a     :     Optional[Any]= "text_model.encoder.layers.22.layer_norm2.bias" in text_enc_dict

	if is_vaa_model:
		# Need to add the tag 'transformer' in advance so we can knock it out from the final layer-norm
		_a     :     Any= {"transformer." + k: v for k, v in text_enc_dict.items()}
		_a     :     List[Any]= convert_text_enc_state_dict_vaa(text_enc_dict)
		_a     :     List[str]= {"cond_stage_model.model." + k: v for k, v in text_enc_dict.items()}
	else:
		_a     :     Tuple= convert_text_enc_state_dict(text_enc_dict)
		_a     :     Any= {"cond_stage_model.transformer." + k: v for k, v in text_enc_dict.items()}

	# Put together new checkpoint
	_a     :     Tuple= {**unet_state_dict, **vae_state_dict, **text_enc_dict}
	if args.half:
		_a     :     Any= {k: v.half() for k, v in state_dict.items()}

	if args.use_safetensors:
		save_file(state_dict, args.checkpoint_path)
	else:
		_a     :     Union[str, Any]= {"state_dict": state_dict}
		torch.save(state_dict, args.checkpoint_path)











import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     0






						@slow
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(GPTaTokenizer, GPTaTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(RobertaTokenizer, RobertaTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	20						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     AutoConfig.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	__lowercase						)
							# Check that tokenizer_type ≠ model_type
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	config=__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with pytest.raises(__lowercase						):
								AutoTokenizer.from_pretrained("./"       ,	tokenizer_type="xxx"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								snake_case_		    =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)

								if isinstance(__lowercase       ,	__lowercase						):
									self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,	__lowercase						)
								else:
									self.assertEqual(tokenizer.do_lower_case       ,	__lowercase						)

								self.assertEqual(tokenizer.model_max_length       ,	5_12						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								with self.assertRaisesRegex(
								    __lowercase       ,	"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"       ,	):
									snake_case_		    =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     TOKENIZER_MAPPING.values()
							snake_case_		    =     []

							for slow_tok, fast_tok in tokenizers:
								if slow_tok is not None:
									tokenizer_names.append(slow_tok.__name__						)

								if fast_tok is not None:
									tokenizer_names.append(fast_tok.__name__						)

							for tokenizer_name in tokenizer_names:
								# must find the right class
								tokenizer_class_from_name(__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"       ,	use_fast=__lowercase						)       ,	__lowercase						)
							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						)       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("distilbert-base-uncased"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     "Hello, world. How are you?"
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)

							snake_case_		    =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)






						@require_tokenizers
						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"						)
							self.assertEqual(type(__lowercase						)       ,	__lowercase						)
							self.assertEqual(tokenizer.model_max_length       ,	5_12						)
							self.assertEqual(tokenizer.vocab_size       ,	3_00_00						)
							self.assertEqual(tokenizer.unk_token       ,	"[UNK]"						)
							self.assertEqual(tokenizer.padding_side       ,	"right"						)
							self.assertEqual(tokenizer.truncation_side       ,	"right"						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase       ,	tokenizer.__class__						)
							self.assertEqual(tokenizera.vocab_size       ,	12						)






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("ctrl"						)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     get_tokenizer_config("bert-base-cased"						)
							snake_case_		    =     config.pop("_commit_hash"       ,	__lowercase						)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(__lowercase       ,	{"do_lower_case": False}						)

							# This model does not have a tokenizer_config so we get back an empty dict.
							snake_case_		    =     get_tokenizer_config(__lowercase						)
							self.assertDictEqual(__lowercase       ,	{}						)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     get_tokenizer_config(__lowercase						)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config["tokenizer_class"]       ,	"BertTokenizer"						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)

								snake_case_		    =     CustomTokenizer.from_pretrained(__lowercase						)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						@require_tokenizers
						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								# Can register in two steps
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, None)						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								del TOKENIZER_MAPPING._extra_content[CustomConfig]
								# Can register in one step
								AutoTokenizer.register(
								    __lowercase       ,	slow_tokenizer_class=__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)

								# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
								# and that model does not have a tokenizer.json
								with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case_		    =     BertTokenizerFast.from_pretrained(__lowercase						)
									bert_tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     CustomTokenizerFast.from_pretrained(__lowercase						)

								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	use_fast=__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)

							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase						)
							self.assertTrue(reloaded_tokenizer.special_attribute_present						)

							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(reloaded_tokenizer.special_attribute_present						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   False







							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   NewTokenizer
													lowerCAmelCase_     		=   False

							try:
								AutoConfig.register("custom"       ,	__lowercase						)
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								# If remote code is not set, the default is to use local
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote code is disabled, we load the local one.
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote is enabled, we load from the Hub
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertTrue(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(tokenizer.special_attribute_present						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(
							    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	"bert-base is not a local folder and is not a valid model identifier"						):
								snake_case_		    =     AutoTokenizer.from_pretrained("bert-base"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	revision="aaaaaa"						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							with RequestCounter() as counter:
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)
