




import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    MegatronBertForCausalLM,
		    MegatronBertForMaskedLM,
		    MegatronBertForMultipleChoice,
		    MegatronBertForNextSentencePrediction,
		    MegatronBertForPreTraining,
		    MegatronBertForQuestionAnswering,
		    MegatronBertForSequenceClassification,
		    MegatronBertForTokenClassification,
		    MegatronBertModel,
		)
class 		UpperCAmelCase   :







						'''simple docstring'''





						def __init__(						self							:     Union[str, Any]       ,	__lowercase							:     Any       ,	__lowercase							:     Any=13       ,	__lowercase							:     Any=7       ,	__lowercase							:     int=True       ,	__lowercase							:     Optional[Any]=True       ,	__lowercase							:     List[str]=True       ,	__lowercase							:     Dict=True       ,	__lowercase							:     Optional[int]=99       ,	__lowercase							:     str=64       ,	__lowercase							:     Optional[int]=32       ,	__lowercase							:     Any=5       ,	__lowercase							:     Union[str, Any]=4       ,	__lowercase							:     Any=37       ,	__lowercase							:     List[Any]="gelu"       ,	__lowercase							:     Dict=0.1       ,	__lowercase							:     Tuple=0.1       ,	__lowercase							:     List[str]=5_12       ,	__lowercase							:     Tuple=16       ,	__lowercase							:     Tuple=2       ,	__lowercase							:     Dict=0.02       ,	__lowercase							:     Any=3       ,	__lowercase							:     List[str]=4       ,	__lowercase							:     Union[str, Any]=None       ,	):






							"""simple docstring"""






							snake_case_		    =     parent
							snake_case_		    =     batch_size
							snake_case_		    =     seq_length
							snake_case_		    =     is_training
							snake_case_		    =     use_input_mask
							snake_case_		    =     use_token_type_ids
							snake_case_		    =     use_labels
							snake_case_		    =     vocab_size
							snake_case_		    =     hidden_size
							snake_case_		    =     embedding_size
							snake_case_		    =     num_hidden_layers
							snake_case_		    =     num_attention_heads
							snake_case_		    =     intermediate_size
							snake_case_		    =     hidden_act
							snake_case_		    =     hidden_dropout_prob
							snake_case_		    =     attention_probs_dropout_prob
							snake_case_		    =     max_position_embeddings
							snake_case_		    =     type_vocab_size
							snake_case_		    =     type_sequence_label_size
							snake_case_		    =     initializer_range
							snake_case_		    =     num_labels
							snake_case_		    =     num_choices
							snake_case_		    =     scope






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size						)

							snake_case_		    =     None
							if self.use_input_mask:
								snake_case_		    =     random_attention_mask([self.batch_size, self.seq_length]						)

							snake_case_		    =     None
							if self.use_token_type_ids:
								snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size						)

							snake_case_		    =     None
							snake_case_		    =     None
							snake_case_		    =     None
							if self.use_labels:
								snake_case_		    =     ids_tensor([self.batch_size]       ,	self.type_sequence_label_size						)
								snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels						)
								snake_case_		    =     ids_tensor([self.batch_size]       ,	self.num_choices						)

							snake_case_		    =     self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							return MegatronBertConfig(
							    vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_size=self.intermediate_size       ,	embedding_size=self.embedding_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout_prob=self.hidden_dropout_prob       ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	is_decoder=__lowercase       ,	initializer_range=self.initializer_range       ,	)






						def        snake_case__						(						self							:     Union[str, Any]       ,	__lowercase							:     Any       ,	__lowercase							:     str       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Tuple       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     int       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase						)
							snake_case_		    =     model(__lowercase       ,	token_type_ids=__lowercase						)
							snake_case_		    =     model(__lowercase						)

							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)						)
							self.parent.assertEqual(result.pooler_output.shape       ,	(self.batch_size, self.hidden_size)						)






						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Any       ,	__lowercase							:     Any						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertForMaskedLM(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)						)






						def        snake_case__						(						self							:     Tuple       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Dict       ,	__lowercase							:     int       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Tuple       ,	__lowercase							:     Any						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertForCausalLM(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)						)






						def        snake_case__						(						self							:     str       ,	__lowercase							:     Any       ,	__lowercase							:     Any       ,	__lowercase							:     str       ,	__lowercase							:     str       ,	__lowercase							:     List[str]       ,	__lowercase							:     List[str]       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertForNextSentencePrediction(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(
							    __lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase       ,	)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, 2)						)






						def        snake_case__						(						self							:     Tuple       ,	__lowercase							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     int       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     List[str]       ,	__lowercase							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertForPreTraining(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(
							    __lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase       ,	next_sentence_label=__lowercase       ,	)
							self.parent.assertEqual(result.prediction_logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)						)
							self.parent.assertEqual(result.seq_relationship_logits.shape       ,	(self.batch_size, 2)						)






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     Tuple       ,	__lowercase							:     Any       ,	__lowercase							:     Any       ,	__lowercase							:     Dict       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertForQuestionAnswering(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(
							    __lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	start_positions=__lowercase       ,	end_positions=__lowercase       ,	)
							self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)						)






						def        snake_case__						(						self							:     List[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     str       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Tuple       ,	__lowercase							:     int       ,	__lowercase							:     int       ,	__lowercase							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     self.num_labels
							snake_case_		    =     MegatronBertForSequenceClassification(__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_labels)						)






						def        snake_case__						(						self							:     int       ,	__lowercase							:     List[str]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     List[str]       ,	__lowercase							:     str       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     Any       ,	__lowercase							:     str						):






							"""simple docstring"""






							snake_case_		    =     self.num_labels
							snake_case_		    =     MegatronBertForTokenClassification(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)						)






						def        snake_case__						(						self							:     Tuple       ,	__lowercase							:     Any       ,	__lowercase							:     int       ,	__lowercase							:     List[str]       ,	__lowercase							:     Tuple       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     self.num_choices
							snake_case_		    =     MegatronBertForMultipleChoice(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     input_ids.unsqueeze(1						).expand(-1       ,	self.num_choices       ,	-1						).contiguous()
							snake_case_		    =     token_type_ids.unsqueeze(1						).expand(-1       ,	self.num_choices       ,	-1						).contiguous()
							snake_case_		    =     input_mask.unsqueeze(1						).expand(-1       ,	self.num_choices       ,	-1						).contiguous()
							snake_case_		    =     model(
							    __lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase       ,	)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_choices)						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     self.prepare_config_and_inputs()
							(
							    (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      
							)		    =     config_and_inputs
							snake_case_		    =     {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict







@require_torch
class 		UpperCAmelCase   (				UpperCAmelCase__       ,   UpperCAmelCase__       ,   unittest.TestCase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   (
						    (
						        MegatronBertModel,
						        MegatronBertForMaskedLM,
						        MegatronBertForCausalLM,
						        MegatronBertForMultipleChoice,
						        MegatronBertForNextSentencePrediction,
						        MegatronBertForPreTraining,
						        MegatronBertForQuestionAnswering,
						        MegatronBertForSequenceClassification,
						        MegatronBertForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						lowerCAmelCase_     		=   (
						    {
						        '''feature-extraction''': MegatronBertModel,
						        '''fill-mask''': MegatronBertForMaskedLM,
						        '''question-answering''': MegatronBertForQuestionAnswering,
						        '''text-classification''': MegatronBertForSequenceClassification,
						        '''text-generation''': MegatronBertForCausalLM,
						        '''token-classification''': MegatronBertForTokenClassification,
						        '''zero-shot''': MegatronBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						lowerCAmelCase_     		=   True
						# test_resize_embeddings = False
						lowerCAmelCase_     		=   False






						def        snake_case__						(						self							:     Any       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     int       ,	__lowercase							:     Dict=False						):






							"""simple docstring"""






							snake_case_		    =     super()._prepare_for_class(__lowercase       ,	__lowercase       ,	return_labels=__lowercase						)

							if return_labels:
								if model_class in get_values(__lowercase						):
									snake_case_		    =     torch.zeros(
									    (self.model_tester.batch_size, self.model_tester.seq_length)       ,	dtype=torch.long       ,	device=__lowercase						)
									snake_case_		    =     torch.zeros(
									    self.model_tester.batch_size       ,	dtype=torch.long       ,	device=__lowercase						)
							return inputs_dict






						def        snake_case__						(						self							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     MegatronBertModelTester(self						)
							snake_case_		    =     ConfigTester(self       ,	config_class=__lowercase       ,	hidden_size=37						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_model(*__lowercase						)






						def        snake_case__						(						self							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_pretraining(*__lowercase						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_question_answering(*__lowercase						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__lowercase						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_token_classification(*__lowercase						)







def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	return torch.tensor(
	    _A     ,	dtype=torch.long     ,	device=_A     ,	)


lowercase__				:     List[str] 		=						1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						@slow
						@unittest.skip("Model is not available."						)
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     "nvidia/megatron-bert-uncased-345m"
							if "MYDIR" in os.environ:
								snake_case_		    =     os.path.join(os.environ["MYDIR"]       ,	__lowercase						)
							snake_case_		    =     MegatronBertModel.from_pretrained(__lowercase						)
							model.to(__lowercase						)
							model.half()
							snake_case_		    =     _long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]						)
							with torch.no_grad():
								snake_case_		    =     model(__lowercase						)[0]
							snake_case_		    =     torch.Size((1, 9, 10_24)						)
							self.assertEqual(output.shape       ,	__lowercase						)

							snake_case_		    =     [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
							for ii in range(3						):
								for jj in range(3						):
									snake_case_		    =     output[0, ii, jj]
									snake_case_		    =     expected[3 * ii + jj]
									snake_case_		    =     "ii={} jj={} a={} b={}".format(__lowercase       ,	__lowercase       ,	__lowercase       ,	__lowercase						)
									self.assertTrue(math.isclose(__lowercase       ,	__lowercase       ,	rel_tol=__lowercase       ,	abs_tol=__lowercase						)       ,	msg=__lowercase						)






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     0






						@slow
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(GPTaTokenizer, GPTaTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(RobertaTokenizer, RobertaTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	20						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     AutoConfig.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	__lowercase						)
							# Check that tokenizer_type ≠ model_type
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	config=__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with pytest.raises(__lowercase						):
								AutoTokenizer.from_pretrained("./"       ,	tokenizer_type="xxx"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								snake_case_		    =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)

								if isinstance(__lowercase       ,	__lowercase						):
									self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,	__lowercase						)
								else:
									self.assertEqual(tokenizer.do_lower_case       ,	__lowercase						)

								self.assertEqual(tokenizer.model_max_length       ,	5_12						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								with self.assertRaisesRegex(
								    __lowercase       ,	"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"       ,	):
									snake_case_		    =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     TOKENIZER_MAPPING.values()
							snake_case_		    =     []

							for slow_tok, fast_tok in tokenizers:
								if slow_tok is not None:
									tokenizer_names.append(slow_tok.__name__						)

								if fast_tok is not None:
									tokenizer_names.append(fast_tok.__name__						)

							for tokenizer_name in tokenizer_names:
								# must find the right class
								tokenizer_class_from_name(__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"       ,	use_fast=__lowercase						)       ,	__lowercase						)
							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						)       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("distilbert-base-uncased"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     "Hello, world. How are you?"
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)

							snake_case_		    =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)






						@require_tokenizers
						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"						)
							self.assertEqual(type(__lowercase						)       ,	__lowercase						)
							self.assertEqual(tokenizer.model_max_length       ,	5_12						)
							self.assertEqual(tokenizer.vocab_size       ,	3_00_00						)
							self.assertEqual(tokenizer.unk_token       ,	"[UNK]"						)
							self.assertEqual(tokenizer.padding_side       ,	"right"						)
							self.assertEqual(tokenizer.truncation_side       ,	"right"						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase       ,	tokenizer.__class__						)
							self.assertEqual(tokenizera.vocab_size       ,	12						)






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("ctrl"						)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     get_tokenizer_config("bert-base-cased"						)
							snake_case_		    =     config.pop("_commit_hash"       ,	__lowercase						)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(__lowercase       ,	{"do_lower_case": False}						)

							# This model does not have a tokenizer_config so we get back an empty dict.
							snake_case_		    =     get_tokenizer_config(__lowercase						)
							self.assertDictEqual(__lowercase       ,	{}						)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     get_tokenizer_config(__lowercase						)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config["tokenizer_class"]       ,	"BertTokenizer"						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)

								snake_case_		    =     CustomTokenizer.from_pretrained(__lowercase						)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						@require_tokenizers
						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								# Can register in two steps
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, None)						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								del TOKENIZER_MAPPING._extra_content[CustomConfig]
								# Can register in one step
								AutoTokenizer.register(
								    __lowercase       ,	slow_tokenizer_class=__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)

								# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
								# and that model does not have a tokenizer.json
								with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case_		    =     BertTokenizerFast.from_pretrained(__lowercase						)
									bert_tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     CustomTokenizerFast.from_pretrained(__lowercase						)

								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	use_fast=__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)

							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase						)
							self.assertTrue(reloaded_tokenizer.special_attribute_present						)

							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(reloaded_tokenizer.special_attribute_present						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   False







							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   NewTokenizer
													lowerCAmelCase_     		=   False

							try:
								AutoConfig.register("custom"       ,	__lowercase						)
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								# If remote code is not set, the default is to use local
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote code is disabled, we load the local one.
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote is enabled, we load from the Hub
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertTrue(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(tokenizer.special_attribute_present						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(
							    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	"bert-base is not a local folder and is not a valid model identifier"						):
								snake_case_		    =     AutoTokenizer.from_pretrained("bert-base"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	revision="aaaaaa"						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							with RequestCounter() as counter:
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)
