'''simple docstring'''

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast







class 				_lowercase     (				unittest.TestCase			):


 '''simple docstring'''






 def a					(      self   :	Optional[int]							)  ->    Any:
  __lowerCAmelCase											=						0
 @slow
 def a					(      self   :	Any							)  ->    Union[str, Any]:
  for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
   self.assertIsNotNone(SCREAMING_SNAKE_CASE__							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (BertTokenizer, BertTokenizerFast)							)
   self.assertGreater(len(SCREAMING_SNAKE_CASE__							)      ,       0							)

  for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
   self.assertIsNotNone(SCREAMING_SNAKE_CASE__							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (GPTaTokenizer, GPTaTokenizerFast)							)
   self.assertGreater(len(SCREAMING_SNAKE_CASE__							)      ,       0							)
 def a					(      self   :	List[str]							)  ->    Optional[int]:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (BertTokenizer, BertTokenizerFast)							)
  self.assertEqual(tokenizer.vocab_size      ,       12							)
 def a					(      self   :	int							)  ->    Optional[Any]:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (RobertaTokenizer, RobertaTokenizerFast)							)
  self.assertEqual(tokenizer.vocab_size      ,       20							)
 def a					(      self   :	int							)  ->    Dict:
  __lowerCAmelCase											=						AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__							)
  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)
  # Check that tokenizer_type ≠ model_type
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       config=SCREAMING_SNAKE_CASE__							)
  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (BertTokenizer, BertTokenizerFast)							)
  self.assertEqual(tokenizer.vocab_size      ,       12							)
 def a					(      self   :	str							)  ->    Any:
  with tempfile.TemporaryDirectory() as tmp_dir:
   shutil.copy("""./tests/fixtures/vocab.txt"""      ,       os.path.join(SCREAMING_SNAKE_CASE__      ,       """vocab.txt"""							)							)

   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       tokenizer_type="""bert"""      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)

  with tempfile.TemporaryDirectory() as tmp_dir:
   shutil.copy("""./tests/fixtures/vocab.json"""      ,       os.path.join(SCREAMING_SNAKE_CASE__      ,       """vocab.json"""							)							)
   shutil.copy("""./tests/fixtures/merges.txt"""      ,       os.path.join(SCREAMING_SNAKE_CASE__      ,       """merges.txt"""							)							)

   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       tokenizer_type="""gpt2"""      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)
 @require_tokenizers
 def a					(      self   :	List[str]							)  ->    Optional[Any]:
  with tempfile.TemporaryDirectory() as tmp_dir:
   shutil.copy("""./tests/fixtures/vocab.txt"""      ,       os.path.join(SCREAMING_SNAKE_CASE__      ,       """vocab.txt"""							)							)

   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       tokenizer_type="""bert"""							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)

  with tempfile.TemporaryDirectory() as tmp_dir:
   shutil.copy("""./tests/fixtures/vocab.json"""      ,       os.path.join(SCREAMING_SNAKE_CASE__      ,       """vocab.json"""							)							)
   shutil.copy("""./tests/fixtures/merges.txt"""      ,       os.path.join(SCREAMING_SNAKE_CASE__      ,       """merges.txt"""							)							)

   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       tokenizer_type="""gpt2"""							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	str							)  ->    Tuple:
  with pytest.raises(SCREAMING_SNAKE_CASE__							):
   AutoTokenizer.from_pretrained("""./"""      ,       tokenizer_type="""xxx"""							)
 @require_tokenizers
 def a					(      self   :	int							)  ->    int:
  for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
   __lowerCAmelCase											=						tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
   self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (BertTokenizer, BertTokenizerFast)							)

   if isinstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							):
    self.assertEqual(tokenizer.basic_tokenizer.do_lower_case      ,       SCREAMING_SNAKE_CASE__							)
   else:
    self.assertEqual(tokenizer.do_lower_case      ,       SCREAMING_SNAKE_CASE__							)

   self.assertEqual(tokenizer.model_max_length      ,       5_12							)
 @require_tokenizers
 def a					(      self   :	Optional[Any]							)  ->    List[str]:
  for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
   with self.assertRaisesRegex(
       SCREAMING_SNAKE_CASE__      ,       """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""      ,       ):
    __lowerCAmelCase											=						tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)
 def a					(      self   :	Tuple							)  ->    int:
  # tests: https://github.com/huggingface/transformers/pull/13251
  # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
  # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
  __lowerCAmelCase											=						TOKENIZER_MAPPING.values()
  __lowerCAmelCase											=						[]

  for slow_tok, fast_tok in tokenizers:
   if slow_tok is not None:
    tokenizer_names.append(slow_tok.__name__							)

   if fast_tok is not None:
    tokenizer_names.append(fast_tok.__name__							)

  for tokenizer_name in tokenizer_names:
   # must find the right class
   tokenizer_class_from_name(SCREAMING_SNAKE_CASE__							)
 @require_tokenizers
 def a					(      self   :	int							)  ->    str:
  self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""      ,       use_fast=SCREAMING_SNAKE_CASE__							)      ,       SCREAMING_SNAKE_CASE__							)
  self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)      ,       SCREAMING_SNAKE_CASE__							)
 @require_tokenizers
 def a					(      self   :	Tuple							)  ->    int:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""distilbert-base-uncased"""      ,       do_lower_case=SCREAMING_SNAKE_CASE__							)
  __lowerCAmelCase											=						"""Hello, world. How are you?"""
  __lowerCAmelCase											=						tokenizer.tokenize(SCREAMING_SNAKE_CASE__							)
  self.assertEqual("""[UNK]"""      ,       tokens[0]							)

  __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""      ,       do_lower_case=SCREAMING_SNAKE_CASE__							)
  __lowerCAmelCase											=						tokenizer.tokenize(SCREAMING_SNAKE_CASE__							)
  self.assertEqual("""[UNK]"""      ,       tokens[0]							)
 @require_tokenizers
 def a					(      self   :	str							)  ->    Any:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
  self.assertEqual(type(SCREAMING_SNAKE_CASE__							)      ,       SCREAMING_SNAKE_CASE__							)
  self.assertEqual(tokenizer.model_max_length      ,       5_12							)
  self.assertEqual(tokenizer.vocab_size      ,       3_00_00							)
  self.assertEqual(tokenizer.unk_token      ,       """[UNK]"""							)
  self.assertEqual(tokenizer.padding_side      ,       """right"""							)
  self.assertEqual(tokenizer.truncation_side      ,       """right"""							)
 def a					(      self   :	Optional[int]							)  ->    int:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       (BertTokenizer, BertTokenizerFast)							)
  with tempfile.TemporaryDirectory() as tmp_dir:
   tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)

  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       tokenizer.__class__							)
  self.assertEqual(tokenizera.vocab_size      ,       12							)
 def a					(      self   :	List[str]							)  ->    Tuple:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""ctrl"""							)
  # There is no fast CTRL so this always gives us a slow tokenizer.
  self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	Any							)  ->    Dict:
  # Check we can load the tokenizer config of an online model.
  __lowerCAmelCase											=						get_tokenizer_config("""bert-base-cased"""							)
  __lowerCAmelCase											=						config.pop("""_commit_hash"""      ,       SCREAMING_SNAKE_CASE__							)
  # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
  self.assertEqual(SCREAMING_SNAKE_CASE__      ,       {"""do_lower_case""": False}							)

  # This model does not have a tokenizer_config so we get back an empty dict.
  __lowerCAmelCase											=						get_tokenizer_config(SCREAMING_SNAKE_CASE__							)
  self.assertDictEqual(SCREAMING_SNAKE_CASE__      ,       {}							)

  # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
  with tempfile.TemporaryDirectory() as tmp_dir:
   tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)
   __lowerCAmelCase											=						get_tokenizer_config(SCREAMING_SNAKE_CASE__							)

  # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
  self.assertEqual(config["""tokenizer_class"""]      ,       """BertTokenizer"""							)
 def a					(      self   :	Optional[int]							)  ->    Dict:
  try:
   AutoConfig.register("""custom"""      ,       SCREAMING_SNAKE_CASE__							)

   AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       slow_tokenizer_class=SCREAMING_SNAKE_CASE__							)
   # Trying to register something existing in the Transformers library will raise an error
   with self.assertRaises(SCREAMING_SNAKE_CASE__							):
    AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       slow_tokenizer_class=SCREAMING_SNAKE_CASE__							)

   __lowerCAmelCase											=						CustomTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)

    __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
    self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)

  finally:
   if "custom" in CONFIG_MAPPING._extra_content:
    del CONFIG_MAPPING._extra_content["custom"]
   if CustomConfig in TOKENIZER_MAPPING._extra_content:
    del TOKENIZER_MAPPING._extra_content[CustomConfig]
 @require_tokenizers
 def a					(      self   :	int							)  ->    Optional[int]:
  try:
   AutoConfig.register("""custom"""      ,       SCREAMING_SNAKE_CASE__							)

   # Can register in two steps
   AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       slow_tokenizer_class=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(TOKENIZER_MAPPING[CustomConfig]      ,       (CustomTokenizer, None)							)
   AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       fast_tokenizer_class=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(TOKENIZER_MAPPING[CustomConfig]      ,       (CustomTokenizer, CustomTokenizerFast)							)

   del TOKENIZER_MAPPING._extra_content[CustomConfig]
   # Can register in one step
   AutoTokenizer.register(
       SCREAMING_SNAKE_CASE__      ,       slow_tokenizer_class=SCREAMING_SNAKE_CASE__      ,       fast_tokenizer_class=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(TOKENIZER_MAPPING[CustomConfig]      ,       (CustomTokenizer, CustomTokenizerFast)							)

   # Trying to register something existing in the Transformers library will raise an error
   with self.assertRaises(SCREAMING_SNAKE_CASE__							):
    AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       fast_tokenizer_class=SCREAMING_SNAKE_CASE__							)

   # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
   # and that model does not have a tokenizer.json
   with tempfile.TemporaryDirectory() as tmp_dir:
    __lowerCAmelCase											=						BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__							)
    bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)
    __lowerCAmelCase											=						CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE__							)

   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)

    __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							)
    self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)

    __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       use_fast=SCREAMING_SNAKE_CASE__							)
    self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)

  finally:
   if "custom" in CONFIG_MAPPING._extra_content:
    del CONFIG_MAPPING._extra_content["custom"]
   if CustomConfig in TOKENIZER_MAPPING._extra_content:
    del TOKENIZER_MAPPING._extra_content[CustomConfig]
 def a					(      self   :	Dict							)  ->    Optional[int]:
  # If remote code is not set, we will time out when asking whether to load the model.
  with self.assertRaises(SCREAMING_SNAKE_CASE__							):
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
  # If remote code is disabled, we can't load this config.
  with self.assertRaises(SCREAMING_SNAKE_CASE__							):
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__							)

  __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__							)
  self.assertTrue(tokenizer.special_attribute_present							)
  # Test tokenizer can be reloaded.
  with tempfile.TemporaryDirectory() as tmp_dir:
   tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       trust_remote_code=SCREAMING_SNAKE_CASE__							)
  self.assertTrue(reloaded_tokenizer.special_attribute_present							)

  if is_tokenizers_available():
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizerFast"""							)
   self.assertEqual(reloaded_tokenizer.__class__.__name__      ,       """NewTokenizerFast"""							)

   # Test we can also load the slow version
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertTrue(tokenizer.special_attribute_present							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
   # Test tokenizer can be reloaded.
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__							)
    __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       trust_remote_code=SCREAMING_SNAKE_CASE__      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(reloaded_tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
   self.assertTrue(reloaded_tokenizer.special_attribute_present							)
  else:
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
   self.assertEqual(reloaded_tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
 @require_tokenizers
 def a					(      self   :	Any							)  ->    List[Any]:







  class 				_lowercase     (				UpperCAmelCase__			):


   '''simple docstring'''






   _SCREAMING_SNAKE_CASE     :       int      	=					False







  class 				_lowercase     (				UpperCAmelCase__			):


   '''simple docstring'''






   _SCREAMING_SNAKE_CASE     :       List[Any]      	=					NewTokenizer
   _SCREAMING_SNAKE_CASE     :       int      	=					False

  try:
   AutoConfig.register("""custom"""      ,       SCREAMING_SNAKE_CASE__							)
   AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       slow_tokenizer_class=SCREAMING_SNAKE_CASE__							)
   AutoTokenizer.register(SCREAMING_SNAKE_CASE__      ,       fast_tokenizer_class=SCREAMING_SNAKE_CASE__							)
   # If remote code is not set, the default is to use local
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizerFast"""							)
   self.assertFalse(tokenizer.special_attribute_present							)
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
   self.assertFalse(tokenizer.special_attribute_present							)

   # If remote code is disabled, we load the local one.
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizerFast"""							)
   self.assertFalse(tokenizer.special_attribute_present							)
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
   self.assertFalse(tokenizer.special_attribute_present							)

   # If remote is enabled, we load from the Hub
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizerFast"""							)
   self.assertTrue(tokenizer.special_attribute_present							)
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
   self.assertTrue(tokenizer.special_attribute_present							)

  finally:
   if "custom" in CONFIG_MAPPING._extra_content:
    del CONFIG_MAPPING._extra_content["custom"]
   if CustomConfig in TOKENIZER_MAPPING._extra_content:
    del TOKENIZER_MAPPING._extra_content[CustomConfig]
 def a					(      self   :	Optional[int]							)  ->    List[str]:
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
      """hf-internal-testing/test_dynamic_tokenizer_legacy"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__							)
  self.assertTrue(tokenizer.special_attribute_present							)
  if is_tokenizers_available():
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizerFast"""							)

   # Test we can also load the slow version
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(
       """hf-internal-testing/test_dynamic_tokenizer_legacy"""      ,       trust_remote_code=SCREAMING_SNAKE_CASE__      ,       use_fast=SCREAMING_SNAKE_CASE__							)
   self.assertTrue(tokenizer.special_attribute_present							)
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
  else:
   self.assertEqual(tokenizer.__class__.__name__      ,       """NewTokenizer"""							)
 def a					(      self   :	str							)  ->    Any:
  with self.assertRaisesRegex(
      SCREAMING_SNAKE_CASE__      ,       """bert-base is not a local folder and is not a valid model identifier"""							):
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""bert-base"""							)
 def a					(      self   :	Dict							)  ->    str:
  with self.assertRaisesRegex(
      SCREAMING_SNAKE_CASE__      ,       R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,       revision="""aaaaaa"""							)





 def a					(      self   :	int							)  ->    int:
  # Make sure we have cached the tokenizer.
  __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
  with RequestCounter() as counter:
   __lowerCAmelCase											=						AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
   self.assertEqual(counter.get_request_count      ,       0							)
   self.assertEqual(counter.head_request_count      ,       1							)
   self.assertEqual(counter.other_request_count      ,       0							)







'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A			:      int					      =				logging.get_logger(__name__)

_A			:      Any					      =				{'''ctrl''': '''https://huggingface.co/ctrl/resolve/main/config.json'''}







class 				_lowercase     (				UpperCAmelCase__			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       Any      	=					"""ctrl"""
 _SCREAMING_SNAKE_CASE     :       Optional[Any]      	=					["""past_key_values"""]
 _SCREAMING_SNAKE_CASE     :       Union[str, Any]      	=					{
     """max_position_embeddings""": """n_positions""",
     """hidden_size""": """n_embd""",
     """num_attention_heads""": """n_head""",
     """num_hidden_layers""": """n_layer""",
 }
 def __init__(      self   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]=24_65_34      ,       SCREAMING_SNAKE_CASE__   :	Optional[int]=2_56      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]=12_80      ,       SCREAMING_SNAKE_CASE__   :	str=81_92      ,       SCREAMING_SNAKE_CASE__   :	List[str]=48      ,       SCREAMING_SNAKE_CASE__   :	Tuple=16      ,       SCREAMING_SNAKE_CASE__   :	str=0.1      ,       SCREAMING_SNAKE_CASE__   :	Dict=0.1      ,       SCREAMING_SNAKE_CASE__   :	Dict=1e-6      ,       SCREAMING_SNAKE_CASE__   :	Any=0.0_2      ,       SCREAMING_SNAKE_CASE__   :	Tuple=True      ,       **SCREAMING_SNAKE_CASE__   :	Optional[int]      ,       )  ->    Tuple:
  __lowerCAmelCase											=						vocab_size
  __lowerCAmelCase											=						n_positions
  __lowerCAmelCase											=						n_embd
  __lowerCAmelCase											=						n_layer
  __lowerCAmelCase											=						n_head
  __lowerCAmelCase											=						dff
  __lowerCAmelCase											=						resid_pdrop
  __lowerCAmelCase											=						embd_pdrop
  __lowerCAmelCase											=						layer_norm_epsilon
  __lowerCAmelCase											=						initializer_range

  __lowerCAmelCase											=						use_cache

  super().__init__(**SCREAMING_SNAKE_CASE__							)






