




"""simple docstring"""



import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE				=       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE				=       {
    'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/config.json',
    'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/config.json',
}




class 	UpperCAmelCase_  (						A__							):
    lowercase__											=      '''xlnet'''
    lowercase__											=      ['''mems''']
    lowercase__											=      {
        '''n_token''': '''vocab_size''',  # Backward compatibility
        '''hidden_size''': '''d_model''',
        '''num_attention_heads''': '''n_head''',
        '''num_hidden_layers''': '''n_layer''',
    }




    def __init__(    self						:       List[str]	, snake_case_						:       Union[str, Any]=32_000	, snake_case_						:       Any=1_024	, snake_case_						:       Dict=24	, snake_case_						:       Optional[Any]=16	, snake_case_						:       List[str]=4_096	, snake_case_						:       int="gelu"	, snake_case_						:       Tuple=True	, snake_case_						:       Union[str, Any]="bi"	, snake_case_						:       Any=0.02	, snake_case_						:       Optional[Any]=1e-12	, snake_case_						:       Optional[Any]=0.1	, snake_case_						:       Optional[int]=512	, snake_case_						:       Tuple=None	, snake_case_						:       Optional[Any]=True	, snake_case_						:       Optional[int]=False	, snake_case_						:       List[Any]=False	, snake_case_						:       int=-1	, snake_case_						:       int=False	, snake_case_						:       Union[str, Any]="last"	, snake_case_						:       Any=True	, snake_case_						:       List[str]="tanh"	, snake_case_						:       Optional[int]=0.1	, snake_case_						:       Optional[int]=5	, snake_case_						:       str=5	, snake_case_						:       List[str]=5	, snake_case_						:       Any=1	, snake_case_						:       List[Any]=2	, **snake_case_						:       Optional[Any]	, )      ->							List[Any]:




     '''simple docstring'''

     A__       =				vocab_size
     A__       =				d_model
     A__       =				n_layer
     A__       =				n_head
     if d_model % n_head != 0:
      raise ValueError(F"""'d_model % n_head' ({d_model % n_head}) should be equal to 0"""      )
     if "d_head" in kwargs:
      if kwargs["d_head"] != d_model // n_head:
       raise ValueError(
           F"""`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})"""      )
     A__       =				d_model // n_head
     A__       =				ff_activation
     A__       =				d_inner
     A__       =				untie_r
     A__       =				attn_type

     A__       =				initializer_range
     A__       =				layer_norm_eps

     A__       =				dropout
     A__       =				mem_len
     A__       =				reuse_len
     A__       =				bi_data
     A__       =				clamp_len
     A__       =				same_length

     A__       =				summary_type
     A__       =				summary_use_proj
     A__       =				summary_activation
     A__       =				summary_last_dropout
     A__       =				start_n_top
     A__       =				end_n_top

     A__       =				bos_token_id
     A__       =				pad_token_id
     A__       =				eos_token_id

     if "use_cache" in kwargs:
      warnings.warn(
          "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
          " instead."	, snake_case_	, )
      A__       =				kwargs["use_cache"]

     A__       =				use_mems_eval
     A__       =				use_mems_train
     super().__init__(pad_token_id=snake_case_	, bos_token_id=snake_case_	, eos_token_id=snake_case_	, **snake_case_      )




    @property
    def 							__magic_name__							(    self						:       Optional[Any]      )      ->							Any:




     '''simple docstring'''

     logger.info(F"""The model {self.model_type} is one of the few models that has no sequence length limit."""      )
     return -1




    @max_position_embeddings.setter
    def 							__magic_name__							(    self						:       Optional[int]	, snake_case_						:       Dict      )      ->							Optional[Any]:




     '''simple docstring'''

     raise NotImplementedError(
         F"""The model {self.model_type} is one of the few models that has no sequence length limit."""      )













import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A__	:			List[Any]			   = logging.get_logger(__name__)

A__	:			str			   = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__	:			int			   = {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A__	:			Optional[Any]			   = {
    'distilbert-base-uncased': 5_12,
    'distilbert-base-uncased-distilled-squad': 5_12,
    'distilbert-base-cased': 5_12,
    'distilbert-base-cased-distilled-squad': 5_12,
    'distilbert-base-german-cased': 5_12,
    'distilbert-base-multilingual-cased': 5_12,
}


A__	:			List[str]			   = {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       VOCAB_FILES_NAMES
				lowercase__											=       PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=       PRETRAINED_INIT_CONFIGURATION
				lowercase__											=       ["""input_ids""", """attention_mask"""]
				lowercase__											=       DistilBertTokenizer




				def __init__(     self						:    List[Any],  lowerCamelCase						:    List[Any]=None,  lowerCamelCase						:    Dict=None,  lowerCamelCase						:    str=True,  lowerCamelCase						:    Optional[int]="[UNK]",  lowerCamelCase						:    Optional[Any]="[SEP]",  lowerCamelCase						:    List[Any]="[PAD]",  lowerCamelCase						:    Any="[CLS]",  lowerCamelCase						:    Union[str, Any]="[MASK]",  lowerCamelCase						:    str=True,  lowerCamelCase						:    int=None,  **lowerCamelCase						:    Union[str, Any],  ):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,  tokenizer_file=lowerCamelCase,  do_lower_case=lowerCamelCase,  unk_token=lowerCamelCase,  sep_token=lowerCamelCase,  pad_token=lowerCamelCase,  cls_token=lowerCamelCase,  mask_token=lowerCamelCase,  tokenize_chinese_chars=lowerCamelCase,  strip_accents=lowerCamelCase,  **lowerCamelCase,  )

								lowercase__    =			json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
								if (
								    normalizer_state.get('''lowercase''',  lowerCamelCase  ) != do_lower_case
								    or normalizer_state.get('''strip_accents''',  lowerCamelCase  ) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars''',  lowerCamelCase  ) != tokenize_chinese_chars
								):
												lowercase__    =			getattr(lowerCamelCase,  normalizer_state.pop('''type'''  )  )
												lowercase__    =			do_lower_case
												lowercase__    =			strip_accents
												lowercase__    =			tokenize_chinese_chars
												lowercase__    =			normalizer_class(**lowerCamelCase  )

								lowercase__    =			do_lower_case




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    Optional[Any],  lowerCamelCase						:    List[Any]=None  ):

								'''simple docstring'''
								lowercase__    =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''
								lowercase__    =			[self.sep_token_id]
								lowercase__    =			[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    str,  lowerCamelCase						:    Optional[str] = None  ):

								'''simple docstring'''
								lowercase__    =			self._tokenizer.model.save(lowerCamelCase,  name=lowerCamelCase  )
								return tuple(lowerCamelCase  )




