
'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


__SCREAMING_SNAKE_CASE  =			logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE  =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__SCREAMING_SNAKE_CASE  =			{
    "vocab_file": {
        "microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json",
    },
}

__SCREAMING_SNAKE_CASE  =			{
    "microsoft/mpnet-base": 512,
}

__SCREAMING_SNAKE_CASE  =			{
    "microsoft/mpnet-base": {"do_lower_case": True},
}




class 						__snake_case (						_a  ):
		a_        =				VOCAB_FILES_NAMES
		a_        =				PRETRAINED_VOCAB_FILES_MAP
		a_        =				PRETRAINED_INIT_CONFIGURATION
		a_        =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a_        =				MPNetTokenizer
		a_        =				["input_ids", "attention_mask"]




		def __init__(     self				:Tuple ,						_SCREAMING_SNAKE_CASE				:Dict=None ,						_SCREAMING_SNAKE_CASE				:List[Any]=None ,						_SCREAMING_SNAKE_CASE				:Optional[Any]=True ,						_SCREAMING_SNAKE_CASE				:Any="<s>" ,						_SCREAMING_SNAKE_CASE				:int="</s>" ,						_SCREAMING_SNAKE_CASE				:Optional[Any]="</s>" ,						_SCREAMING_SNAKE_CASE				:Any="<s>" ,						_SCREAMING_SNAKE_CASE				:Optional[int]="[UNK]" ,						_SCREAMING_SNAKE_CASE				:List[Any]="<pad>" ,						_SCREAMING_SNAKE_CASE				:int="<mask>" ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=True ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=None ,						**_SCREAMING_SNAKE_CASE				:Optional[int] ,						)				-> Optional[Any]:






								'''simple docstring'''






								super().__init__(
								    _SCREAMING_SNAKE_CASE ,						tokenizer_file=_SCREAMING_SNAKE_CASE ,						do_lower_case=_SCREAMING_SNAKE_CASE ,						bos_token=_SCREAMING_SNAKE_CASE ,						eos_token=_SCREAMING_SNAKE_CASE ,						sep_token=_SCREAMING_SNAKE_CASE ,						cls_token=_SCREAMING_SNAKE_CASE ,						unk_token=_SCREAMING_SNAKE_CASE ,						pad_token=_SCREAMING_SNAKE_CASE ,						mask_token=_SCREAMING_SNAKE_CASE ,						tokenize_chinese_chars=_SCREAMING_SNAKE_CASE ,						strip_accents=_SCREAMING_SNAKE_CASE ,						**_SCREAMING_SNAKE_CASE ,						)

								UpperCamelCase_    :  Dict				    =    json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    pre_tok_state.get("""lowercase""" ,						_SCREAMING_SNAKE_CASE) != do_lower_case
								    or pre_tok_state.get("""strip_accents""" ,						_SCREAMING_SNAKE_CASE) != strip_accents
								):
														UpperCamelCase_    :  Any				    =    getattr(_SCREAMING_SNAKE_CASE ,						pre_tok_state.pop("""type"""))
														UpperCamelCase_    :  Optional[int]				    =    do_lower_case
														UpperCamelCase_    :  str				    =    strip_accents
														UpperCamelCase_    :  str				    =    pre_tok_class(**_SCREAMING_SNAKE_CASE)

								UpperCamelCase_    :  List[str]				    =    do_lower_case




		@property
		def   _snake_case  (     self				:Optional[int])				-> str:






								'''simple docstring'''






								if self._mask_token is None:
														if self.verbose:
																				logger.error("""Using mask_token, but it is not set yet.""")
														return None
								return str(self._mask_token)




		@mask_token.setter
		def   _snake_case  (     self				:Optional[Any] ,						_SCREAMING_SNAKE_CASE				:List[Any])				-> int:






								'''simple docstring'''






								UpperCamelCase_    :  Tuple				    =    AddedToken(_SCREAMING_SNAKE_CASE ,						lstrip=_SCREAMING_SNAKE_CASE ,						rstrip=_SCREAMING_SNAKE_CASE) if isinstance(_SCREAMING_SNAKE_CASE ,						_SCREAMING_SNAKE_CASE) else value
								UpperCamelCase_    :  int				    =    value




		def   _snake_case  (     self				:str ,						_SCREAMING_SNAKE_CASE				:int ,						_SCREAMING_SNAKE_CASE				:Any=None)				-> str:






								'''simple docstring'''






								UpperCamelCase_    :  Tuple				    =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
														return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




		def   _snake_case  (     self				:Optional[Any] ,						_SCREAMING_SNAKE_CASE				:List[int] ,						_SCREAMING_SNAKE_CASE				:Optional[List[int]] = None)				-> List[int]:






								'''simple docstring'''






								UpperCamelCase_    :  Optional[Any]				    =    [self.sep_token_id]
								UpperCamelCase_    :  Dict				    =    [self.cls_token_id]

								if token_ids_a is None:
														return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





		def   _snake_case  (     self				:List[str] ,						_SCREAMING_SNAKE_CASE				:str ,						_SCREAMING_SNAKE_CASE				:Optional[str] = None)				-> Tuple[str]:






								'''simple docstring'''






								UpperCamelCase_    :  List[str]				    =    self._tokenizer.model.save(_SCREAMING_SNAKE_CASE ,						name=_SCREAMING_SNAKE_CASE)
								return tuple(_SCREAMING_SNAKE_CASE)






'''simple docstring'''


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__SCREAMING_SNAKE_CASE  =			logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE  =			{
    "gpt2": "https://huggingface.co/gpt2/resolve/main/config.json",
    "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/config.json",
    "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/config.json",
    "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/config.json",
    "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/config.json",
}




class 						__snake_case (						_a  ):
		a_        =				"gpt2"
		a_        =				["past_key_values"]
		a_        =				{
		    "hidden_size": "n_embd",
		    "max_position_embeddings": "n_positions",
		    "num_attention_heads": "n_head",
		    "num_hidden_layers": "n_layer",
		}




		def __init__(     self				:Union[str, Any] ,						_SCREAMING_SNAKE_CASE				:Dict=5_0_2_5_7 ,						_SCREAMING_SNAKE_CASE				:str=1_0_2_4 ,						_SCREAMING_SNAKE_CASE				:Dict=7_6_8 ,						_SCREAMING_SNAKE_CASE				:List[Any]=1_2 ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=1_2 ,						_SCREAMING_SNAKE_CASE				:Any=None ,						_SCREAMING_SNAKE_CASE				:Any="gelu_new" ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=0.1 ,						_SCREAMING_SNAKE_CASE				:List[str]=0.1 ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=0.1 ,						_SCREAMING_SNAKE_CASE				:str=1E-5 ,						_SCREAMING_SNAKE_CASE				:str=0.02 ,						_SCREAMING_SNAKE_CASE				:Tuple="cls_index" ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=True ,						_SCREAMING_SNAKE_CASE				:Dict=None ,						_SCREAMING_SNAKE_CASE				:Tuple=True ,						_SCREAMING_SNAKE_CASE				:Dict=0.1 ,						_SCREAMING_SNAKE_CASE				:List[Any]=True ,						_SCREAMING_SNAKE_CASE				:Optional[Any]=True ,						_SCREAMING_SNAKE_CASE				:Tuple=5_0_2_5_6 ,						_SCREAMING_SNAKE_CASE				:Any=5_0_2_5_6 ,						_SCREAMING_SNAKE_CASE				:List[str]=False ,						_SCREAMING_SNAKE_CASE				:Any=False ,						**_SCREAMING_SNAKE_CASE				:List[Any] ,						)				-> Optional[Any]:






								'''simple docstring'''






								UpperCamelCase_    :  Optional[Any]				    =    vocab_size
								UpperCamelCase_    :  int				    =    n_positions
								UpperCamelCase_    :  Union[str, Any]				    =    n_embd
								UpperCamelCase_    :  Tuple				    =    n_layer
								UpperCamelCase_    :  Optional[Any]				    =    n_head
								UpperCamelCase_    :  List[Any]				    =    n_inner
								UpperCamelCase_    :  List[Any]				    =    activation_function
								UpperCamelCase_    :  Tuple				    =    resid_pdrop
								UpperCamelCase_    :  Optional[int]				    =    embd_pdrop
								UpperCamelCase_    :  Optional[Any]				    =    attn_pdrop
								UpperCamelCase_    :  List[str]				    =    layer_norm_epsilon
								UpperCamelCase_    :  Tuple				    =    initializer_range
								UpperCamelCase_    :  Union[str, Any]				    =    summary_type
								UpperCamelCase_    :  List[Any]				    =    summary_use_proj
								UpperCamelCase_    :  Tuple				    =    summary_activation
								UpperCamelCase_    :  Any				    =    summary_first_dropout
								UpperCamelCase_    :  Union[str, Any]				    =    summary_proj_to_labels
								UpperCamelCase_    :  Any				    =    scale_attn_weights
								UpperCamelCase_    :  List[str]				    =    use_cache
								UpperCamelCase_    :  int				    =    scale_attn_by_inverse_layer_idx
								UpperCamelCase_    :  int				    =    reorder_and_upcast_attn

								UpperCamelCase_    :  Union[str, Any]				    =    bos_token_id
								UpperCamelCase_    :  Optional[int]				    =    eos_token_id

								super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE ,						eos_token_id=_SCREAMING_SNAKE_CASE ,						**_SCREAMING_SNAKE_CASE)







class 						__snake_case (						_a  ):




		def __init__(     self				:List[Any] ,						_SCREAMING_SNAKE_CASE				:PretrainedConfig ,						_SCREAMING_SNAKE_CASE				:str = "default" ,						_SCREAMING_SNAKE_CASE				:List[PatchingSpec] = None ,						_SCREAMING_SNAKE_CASE				:bool = False ,						)				-> Any:






								'''simple docstring'''






								super().__init__(_SCREAMING_SNAKE_CASE ,						task=_SCREAMING_SNAKE_CASE ,						patching_specs=_SCREAMING_SNAKE_CASE ,						use_past=_SCREAMING_SNAKE_CASE)
								if not getattr(self._config ,						"""pad_token_id""" ,						_SCREAMING_SNAKE_CASE):
														# TODO: how to do that better?
														UpperCamelCase_    :  List[str]				    =    0




		@property
		def   _snake_case  (     self				:Optional[int])				-> Mapping[str, Mapping[int, str]]:






								'''simple docstring'''






								UpperCamelCase_    :  str				    =    OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}})
								if self.use_past:
														self.fill_with_past_key_values_(_SCREAMING_SNAKE_CASE ,						direction="""inputs""")
														UpperCamelCase_    :  Any				    =    {0: """batch""", 1: """past_sequence + sequence"""}
								else:
														UpperCamelCase_    :  Optional[int]				    =    {0: """batch""", 1: """sequence"""}

								return common_inputs




		@property
		def   _snake_case  (     self				:Optional[int])				-> int:






								'''simple docstring'''






								return self._config.n_layer




		@property
		def   _snake_case  (     self				:Optional[int])				-> int:






								'''simple docstring'''






								return self._config.n_head




		def   _snake_case  (     self				:Optional[Any] ,						_SCREAMING_SNAKE_CASE				:PreTrainedTokenizer ,						_SCREAMING_SNAKE_CASE				:int = -1 ,						_SCREAMING_SNAKE_CASE				:int = -1 ,						_SCREAMING_SNAKE_CASE				:bool = False ,						_SCREAMING_SNAKE_CASE				:Optional[TensorType] = None ,						)				-> Mapping[str, Any]:






								'''simple docstring'''






								UpperCamelCase_    :  List[str]				    =    super(_SCREAMING_SNAKE_CASE ,						self).generate_dummy_inputs(
								    _SCREAMING_SNAKE_CASE ,						batch_size=_SCREAMING_SNAKE_CASE ,						seq_length=_SCREAMING_SNAKE_CASE ,						is_pair=_SCREAMING_SNAKE_CASE ,						framework=_SCREAMING_SNAKE_CASE)

								# We need to order the input in the way they appears in the forward()
								UpperCamelCase_    :  Optional[Any]				    =    OrderedDict({"""input_ids""": common_inputs["""input_ids"""]})

								# Need to add the past_keys
								if self.use_past:
														if not is_torch_available():
																				raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed.""")
														else:
																				import torch

																				UpperCamelCase_    ,UpperCamelCase_    :  str				    =    common_inputs["""input_ids"""].shape
																				# Not using the same length for past_key_values
																				UpperCamelCase_    :  Optional[int]				    =    seqlen + 2
																				UpperCamelCase_    :  Tuple				    =    (
																				    batch,
																				    self.num_attention_heads,
																				    past_key_values_length,
																				    self._config.hidden_size // self.num_attention_heads,
																				)
																				UpperCamelCase_    :  List[str]				    =    [
																				    (torch.zeros(_SCREAMING_SNAKE_CASE), torch.zeros(_SCREAMING_SNAKE_CASE)) for _ in range(self.num_layers)
																				]

								UpperCamelCase_    :  Optional[Any]				    =    common_inputs["""attention_mask"""]
								if self.use_past:
														UpperCamelCase_    :  Optional[Any]				    =    ordered_inputs["""attention_mask"""].dtype
														UpperCamelCase_    :  Tuple				    =    torch.cat(
														    [ordered_inputs["""attention_mask"""], torch.ones(_SCREAMING_SNAKE_CASE ,						_SCREAMING_SNAKE_CASE ,						dtype=_SCREAMING_SNAKE_CASE)] ,						dim=1)

								return ordered_inputs





		@property
		def   _snake_case  (     self				:int)				-> int:






								'''simple docstring'''






								return 1_3




