




import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin
class 		UpperCAmelCase   (				UpperCAmelCase__       ,   UpperCAmelCase__			):







						'''simple docstring'''





						@register_to_config
						def __init__(						self							:     Tuple       ,	__lowercase							:     int = 1_28       ,	__lowercase							:     int = 2_56       ,	__lowercase							:     float = 2000.0       ,	__lowercase							:     int = 7_68       ,	__lowercase							:     int = 12       ,	__lowercase							:     int = 12       ,	__lowercase							:     int = 64       ,	__lowercase							:     int = 20_48       ,	__lowercase							:     float = 0.1       ,	):






							"""simple docstring"""






							super().__init__()

							snake_case_		    =     nn.Sequential(
							    nn.Linear(__lowercase       ,	d_model * 4       ,	bias=__lowercase						)       ,	nn.SiLU()       ,	nn.Linear(d_model * 4       ,	d_model * 4       ,	bias=__lowercase						)       ,	nn.SiLU()       ,	)

							snake_case_		    =     nn.Embedding(__lowercase       ,	__lowercase						)
							snake_case_		    =     False

							snake_case_		    =     nn.Linear(__lowercase       ,	__lowercase       ,	bias=__lowercase						)

							snake_case_		    =     nn.Dropout(p=__lowercase						)

							snake_case_		    =     nn.ModuleList()
							for lyr_num in range(__lowercase						):
								# FiLM conditional T5 decoder
								snake_case_		    =     DecoderLayer(d_model=__lowercase       ,	d_kv=__lowercase       ,	num_heads=__lowercase       ,	d_ff=__lowercase       ,	dropout_rate=__lowercase						)
								self.decoders.append(__lowercase						)

							snake_case_		    =     TaLayerNorm(__lowercase						)

							snake_case_		    =     nn.Dropout(p=__lowercase						)
							snake_case_		    =     nn.Linear(__lowercase       ,	__lowercase       ,	bias=__lowercase						)






						def        snake_case__						(						self							:     int       ,	__lowercase							:     int       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     torch.mul(query_input.unsqueeze(-1						)       ,	key_input.unsqueeze(-2						)						)
							return mask.unsqueeze(-3						)






						def        snake_case__						(						self							:     Any       ,	__lowercase							:     Tuple       ,	__lowercase							:     List[str]       ,	__lowercase							:     List[str]						):






							"""simple docstring"""






							snake_case_    ,      snake_case_    ,      snake_case_		    =     decoder_input_tokens.shape
							assert decoder_noise_time.shape == (batch,)

							# decoder_noise_time is in [0, 1), so rescale to expected timing range.
							snake_case_		    =     get_timestep_embedding(
							    decoder_noise_time * self.config.max_decoder_noise_time       ,	embedding_dim=self.config.d_model       ,	max_period=self.config.max_decoder_noise_time       ,	).to(dtype=self.dtype						)

							snake_case_		    =     self.conditioning_emb(__lowercase						).unsqueeze(1						)

							assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

							snake_case_		    =     decoder_input_tokens.shape[1]

							# If we want to use relative positions for audio context, we can just offset
							# this sequence by the length of encodings_and_masks.
							snake_case_		    =     torch.broadcast_to(
							    torch.arange(__lowercase       ,	device=decoder_input_tokens.device						)       ,	(batch, seq_length)       ,	)

							snake_case_		    =     self.position_encoding(__lowercase						)

							snake_case_		    =     self.continuous_inputs_projection(__lowercase						)
							inputs += position_encodings
							snake_case_		    =     self.dropout(__lowercase						)

							# decoder: No padding present.
							snake_case_		    =     torch.ones(
							    decoder_input_tokens.shape[:2]       ,	device=decoder_input_tokens.device       ,	dtype=inputs.dtype						)

							# Translate encoding masks to encoder-decoder masks.
							snake_case_		    =     [(x, self.encoder_decoder_mask(__lowercase       ,	__lowercase						)) for x, y in encodings_and_masks]

							# cross attend style: concat encodings
							snake_case_		    =     torch.cat([x[0] for x in encodings_and_encdec_masks]       ,	dim=1						)
							snake_case_		    =     torch.cat([x[1] for x in encodings_and_encdec_masks]       ,	dim=-1						)

							for lyr in self.decoders:
								snake_case_		    =     lyr(
								    __lowercase       ,	conditioning_emb=__lowercase       ,	encoder_hidden_states=__lowercase       ,	encoder_attention_mask=__lowercase       ,	)[0]

							snake_case_		    =     self.decoder_norm(__lowercase						)
							snake_case_		    =     self.post_dropout(__lowercase						)

							snake_case_		    =     self.spec_out(__lowercase						)
							return spec_out







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     List[Any]       ,	__lowercase							:     str       ,	__lowercase							:     Dict       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     int       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     Tuple=1E-6						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     nn.ModuleList()

							# cond self attention: layer 0
							self.layer.append(
							    TaLayerSelfAttentionCond(d_model=__lowercase       ,	d_kv=__lowercase       ,	num_heads=__lowercase       ,	dropout_rate=__lowercase						)						)

							# cross attention: layer 1
							self.layer.append(
							    TaLayerCrossAttention(
							        d_model=__lowercase       ,	d_kv=__lowercase       ,	num_heads=__lowercase       ,	dropout_rate=__lowercase       ,	layer_norm_epsilon=__lowercase       ,	)						)

							# Film Cond MLP + dropout: last layer
							self.layer.append(
							    TaLayerFFCond(d_model=__lowercase       ,	d_ff=__lowercase       ,	dropout_rate=__lowercase       ,	layer_norm_epsilon=__lowercase						)						)






						def        snake_case__						(						self							:     List[Any]       ,	__lowercase							:     Tuple       ,	__lowercase							:     int=None       ,	__lowercase							:     Optional[Any]=None       ,	__lowercase							:     Dict=None       ,	__lowercase							:     Dict=None       ,	__lowercase							:     str=None       ,	):






							"""simple docstring"""






							snake_case_		    =     self.layer[0](
							    __lowercase       ,	conditioning_emb=__lowercase       ,	attention_mask=__lowercase       ,	)

							if encoder_hidden_states is not None:
								snake_case_		    =     torch.where(encoder_attention_mask > 0       ,	0       ,	-1E10						).to(
								    encoder_hidden_states.dtype						)

								snake_case_		    =     self.layer[1](
								    __lowercase       ,	key_value_states=__lowercase       ,	attention_mask=__lowercase       ,	)

							# Apply Film Conditional Feed Forward layer
							snake_case_		    =     self.layer[-1](__lowercase       ,	__lowercase						)

							return (hidden_states,)







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     List[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     List[Any]       ,	__lowercase							:     str       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     TaLayerNorm(__lowercase						)
							snake_case_		    =     TaFiLMLayer(in_features=d_model * 4       ,	out_features=__lowercase						)
							snake_case_		    =     Attention(query_dim=__lowercase       ,	heads=__lowercase       ,	dim_head=__lowercase       ,	out_bias=__lowercase       ,	scale_qk=__lowercase						)
							snake_case_		    =     nn.Dropout(__lowercase						)






						def        snake_case__						(						self							:     str       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     Union[str, Any]=None       ,	__lowercase							:     Optional[int]=None       ,	):






							"""simple docstring"""






							snake_case_		    =     self.layer_norm(__lowercase						)

							if conditioning_emb is not None:
								snake_case_		    =     self.FiLMLayer(__lowercase       ,	__lowercase						)

							# Self-attention block
							snake_case_		    =     self.attention(__lowercase						)

							snake_case_		    =     hidden_states + self.dropout(__lowercase						)

							return hidden_states







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     Optional[Any]       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     Any       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Any						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     Attention(query_dim=__lowercase       ,	heads=__lowercase       ,	dim_head=__lowercase       ,	out_bias=__lowercase       ,	scale_qk=__lowercase						)
							snake_case_		    =     TaLayerNorm(__lowercase       ,	eps=__lowercase						)
							snake_case_		    =     nn.Dropout(__lowercase						)






						def        snake_case__						(						self							:     Optional[Any]       ,	__lowercase							:     str       ,	__lowercase							:     Tuple=None       ,	__lowercase							:     str=None       ,	):






							"""simple docstring"""






							snake_case_		    =     self.layer_norm(__lowercase						)
							snake_case_		    =     self.attention(
							    __lowercase       ,	encoder_hidden_states=__lowercase       ,	attention_mask=attention_mask.squeeze(1						)       ,	)
							snake_case_		    =     hidden_states + self.dropout(__lowercase						)
							return layer_output







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     List[Any]       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     List[Any]       ,	__lowercase							:     List[Any]       ,	__lowercase							:     List[Any]						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     TaDenseGatedActDense(d_model=__lowercase       ,	d_ff=__lowercase       ,	dropout_rate=__lowercase						)
							snake_case_		    =     TaFiLMLayer(in_features=d_model * 4       ,	out_features=__lowercase						)
							snake_case_		    =     TaLayerNorm(__lowercase       ,	eps=__lowercase						)
							snake_case_		    =     nn.Dropout(__lowercase						)






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[str]       ,	__lowercase							:     Dict=None						):






							"""simple docstring"""






							snake_case_		    =     self.layer_norm(__lowercase						)
							if conditioning_emb is not None:
								snake_case_		    =     self.film(__lowercase       ,	__lowercase						)

							snake_case_		    =     self.DenseReluDense(__lowercase						)
							snake_case_		    =     hidden_states + self.dropout(__lowercase						)
							return hidden_states







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     Optional[Any]       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Any       ,	__lowercase							:     List[str]						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     nn.Linear(__lowercase       ,	__lowercase       ,	bias=__lowercase						)
							snake_case_		    =     nn.Linear(__lowercase       ,	__lowercase       ,	bias=__lowercase						)
							snake_case_		    =     nn.Linear(__lowercase       ,	__lowercase       ,	bias=__lowercase						)
							snake_case_		    =     nn.Dropout(__lowercase						)
							snake_case_		    =     NewGELUActivation()






						def        snake_case__						(						self							:     Optional[Any]       ,	__lowercase							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.act(self.wi_a(__lowercase						)						)
							snake_case_		    =     self.wi_a(__lowercase						)
							snake_case_		    =     hidden_gelu * hidden_linear
							snake_case_		    =     self.dropout(__lowercase						)

							snake_case_		    =     self.wo(__lowercase						)
							return hidden_states







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     Any       ,	__lowercase							:     Dict       ,	__lowercase							:     Any=1E-6						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     nn.Parameter(torch.ones(__lowercase						)						)
							snake_case_		    =     eps






						def        snake_case__						(						self							:     Union[str, Any]       ,	__lowercase							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     hidden_states.to(torch.floataa						).pow(2						).mean(-1       ,	keepdim=__lowercase						)
							snake_case_		    =     hidden_states * torch.rsqrt(variance + self.variance_epsilon						)

							# convert into half-precision if necessary
							if self.weight.dtype in [torch.floataa, torch.bfloataa]:
								snake_case_		    =     hidden_states.to(self.weight.dtype						)

							return self.weight * hidden_states







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def        snake_case__						(						self							:     List[Any]       ,	__lowercase							:     torch.Tensor						):






							"""simple docstring"""






							return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi						) * (input + 0.04_4715 * torch.pow(__lowercase       ,	3.0						))						))







class 		UpperCAmelCase   (				nn.Module			):







						'''simple docstring'''





						def __init__(						self							:     Tuple       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							super().__init__()
							snake_case_		    =     nn.Linear(__lowercase       ,	out_features * 2       ,	bias=__lowercase						)






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     int       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     self.scale_bias(__lowercase						)
							snake_case_    ,      snake_case_		    =     torch.chunk(__lowercase       ,	2       ,	-1						)
							snake_case_		    =     x * (1 + scale) + shift
							return x






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     0






						@slow
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(GPTaTokenizer, GPTaTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(RobertaTokenizer, RobertaTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	20						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     AutoConfig.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	__lowercase						)
							# Check that tokenizer_type ≠ model_type
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	config=__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with pytest.raises(__lowercase						):
								AutoTokenizer.from_pretrained("./"       ,	tokenizer_type="xxx"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								snake_case_		    =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)

								if isinstance(__lowercase       ,	__lowercase						):
									self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,	__lowercase						)
								else:
									self.assertEqual(tokenizer.do_lower_case       ,	__lowercase						)

								self.assertEqual(tokenizer.model_max_length       ,	5_12						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								with self.assertRaisesRegex(
								    __lowercase       ,	"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"       ,	):
									snake_case_		    =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     TOKENIZER_MAPPING.values()
							snake_case_		    =     []

							for slow_tok, fast_tok in tokenizers:
								if slow_tok is not None:
									tokenizer_names.append(slow_tok.__name__						)

								if fast_tok is not None:
									tokenizer_names.append(fast_tok.__name__						)

							for tokenizer_name in tokenizer_names:
								# must find the right class
								tokenizer_class_from_name(__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"       ,	use_fast=__lowercase						)       ,	__lowercase						)
							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						)       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("distilbert-base-uncased"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     "Hello, world. How are you?"
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)

							snake_case_		    =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)






						@require_tokenizers
						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"						)
							self.assertEqual(type(__lowercase						)       ,	__lowercase						)
							self.assertEqual(tokenizer.model_max_length       ,	5_12						)
							self.assertEqual(tokenizer.vocab_size       ,	3_00_00						)
							self.assertEqual(tokenizer.unk_token       ,	"[UNK]"						)
							self.assertEqual(tokenizer.padding_side       ,	"right"						)
							self.assertEqual(tokenizer.truncation_side       ,	"right"						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase       ,	tokenizer.__class__						)
							self.assertEqual(tokenizera.vocab_size       ,	12						)






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("ctrl"						)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     get_tokenizer_config("bert-base-cased"						)
							snake_case_		    =     config.pop("_commit_hash"       ,	__lowercase						)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(__lowercase       ,	{"do_lower_case": False}						)

							# This model does not have a tokenizer_config so we get back an empty dict.
							snake_case_		    =     get_tokenizer_config(__lowercase						)
							self.assertDictEqual(__lowercase       ,	{}						)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     get_tokenizer_config(__lowercase						)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config["tokenizer_class"]       ,	"BertTokenizer"						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)

								snake_case_		    =     CustomTokenizer.from_pretrained(__lowercase						)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						@require_tokenizers
						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								# Can register in two steps
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, None)						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								del TOKENIZER_MAPPING._extra_content[CustomConfig]
								# Can register in one step
								AutoTokenizer.register(
								    __lowercase       ,	slow_tokenizer_class=__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)

								# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
								# and that model does not have a tokenizer.json
								with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case_		    =     BertTokenizerFast.from_pretrained(__lowercase						)
									bert_tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     CustomTokenizerFast.from_pretrained(__lowercase						)

								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	use_fast=__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)

							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase						)
							self.assertTrue(reloaded_tokenizer.special_attribute_present						)

							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(reloaded_tokenizer.special_attribute_present						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   False







							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   NewTokenizer
													lowerCAmelCase_     		=   False

							try:
								AutoConfig.register("custom"       ,	__lowercase						)
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								# If remote code is not set, the default is to use local
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote code is disabled, we load the local one.
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote is enabled, we load from the Hub
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertTrue(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(tokenizer.special_attribute_present						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(
							    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	"bert-base is not a local folder and is not a valid model identifier"						):
								snake_case_		    =     AutoTokenizer.from_pretrained("bert-base"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	revision="aaaaaa"						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							with RequestCounter() as counter:
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)
