
'''simple docstring'''


import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
	from transformers.models.electra.modeling_flax_electra import (
	    FlaxElectraForCausalLM,
	    FlaxElectraForMaskedLM,
	    FlaxElectraForMultipleChoice,
	    FlaxElectraForPreTraining,
	    FlaxElectraForQuestionAnswering,
	    FlaxElectraForSequenceClassification,
	    FlaxElectraForTokenClassification,
	    FlaxElectraModel,
	)




class 						__snake_case (						unittest.TestCase  ):




		def __init__(     self				:Dict ,						_SCREAMING_SNAKE_CASE				:List[Any] ,						_SCREAMING_SNAKE_CASE				:str=1_3 ,						_SCREAMING_SNAKE_CASE				:int=7 ,						_SCREAMING_SNAKE_CASE				:Dict=True ,						_SCREAMING_SNAKE_CASE				:Dict=True ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=True ,						_SCREAMING_SNAKE_CASE				:str=True ,						_SCREAMING_SNAKE_CASE				:List[str]=9_9 ,						_SCREAMING_SNAKE_CASE				:Optional[int]=2_4 ,						_SCREAMING_SNAKE_CASE				:Any=3_2 ,						_SCREAMING_SNAKE_CASE				:List[str]=5 ,						_SCREAMING_SNAKE_CASE				:Dict=4 ,						_SCREAMING_SNAKE_CASE				:int=3_7 ,						_SCREAMING_SNAKE_CASE				:str="gelu" ,						_SCREAMING_SNAKE_CASE				:Tuple=0.1 ,						_SCREAMING_SNAKE_CASE				:Optional[int]=0.1 ,						_SCREAMING_SNAKE_CASE				:int=5_1_2 ,						_SCREAMING_SNAKE_CASE				:Dict=1_6 ,						_SCREAMING_SNAKE_CASE				:int=2 ,						_SCREAMING_SNAKE_CASE				:str=0.02 ,						_SCREAMING_SNAKE_CASE				:List[Any]=4 ,						)				-> str:






								'''simple docstring'''






								UpperCamelCase_    :  Union[str, Any]				    =    parent
								UpperCamelCase_    :  Dict				    =    batch_size
								UpperCamelCase_    :  List[str]				    =    seq_length
								UpperCamelCase_    :  Dict				    =    is_training
								UpperCamelCase_    :  Dict				    =    use_attention_mask
								UpperCamelCase_    :  Tuple				    =    use_token_type_ids
								UpperCamelCase_    :  Any				    =    use_labels
								UpperCamelCase_    :  Optional[Any]				    =    vocab_size
								UpperCamelCase_    :  Tuple				    =    hidden_size
								UpperCamelCase_    :  int				    =    embedding_size
								UpperCamelCase_    :  Optional[Any]				    =    num_hidden_layers
								UpperCamelCase_    :  Any				    =    num_attention_heads
								UpperCamelCase_    :  Optional[Any]				    =    intermediate_size
								UpperCamelCase_    :  Optional[Any]				    =    hidden_act
								UpperCamelCase_    :  Optional[int]				    =    hidden_dropout_prob
								UpperCamelCase_    :  Tuple				    =    attention_probs_dropout_prob
								UpperCamelCase_    :  Optional[int]				    =    max_position_embeddings
								UpperCamelCase_    :  Any				    =    type_vocab_size
								UpperCamelCase_    :  Union[str, Any]				    =    type_sequence_label_size
								UpperCamelCase_    :  List[Any]				    =    initializer_range
								UpperCamelCase_    :  Optional[int]				    =    num_choices




		def   _snake_case  (     self				:str)				-> List[Any]:






								'''simple docstring'''






								UpperCamelCase_    :  Dict				    =    ids_tensor([self.batch_size, self.seq_length] ,						self.vocab_size)

								UpperCamelCase_    :  Tuple				    =    None
								if self.use_attention_mask:
														UpperCamelCase_    :  Dict				    =    random_attention_mask([self.batch_size, self.seq_length])

								UpperCamelCase_    :  int				    =    None
								if self.use_token_type_ids:
														UpperCamelCase_    :  Tuple				    =    ids_tensor([self.batch_size, self.seq_length] ,						self.type_vocab_size)

								UpperCamelCase_    :  int				    =    ElectraConfig(
								    vocab_size=self.vocab_size ,						hidden_size=self.hidden_size ,						embedding_size=self.embedding_size ,						num_hidden_layers=self.num_hidden_layers ,						num_attention_heads=self.num_attention_heads ,						intermediate_size=self.intermediate_size ,						hidden_act=self.hidden_act ,						hidden_dropout_prob=self.hidden_dropout_prob ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob ,						max_position_embeddings=self.max_position_embeddings ,						type_vocab_size=self.type_vocab_size ,						initializer_range=self.initializer_range ,						)

								return config, input_ids, token_type_ids, attention_mask





		def   _snake_case  (     self				:Any)				-> List[Any]:






								'''simple docstring'''






								UpperCamelCase_    :  Union[str, Any]				    =    self.prepare_config_and_inputs()
								UpperCamelCase_    ,UpperCamelCase_    ,UpperCamelCase_    ,UpperCamelCase_    :  Optional[Any]				    =    config_and_inputs
								UpperCamelCase_    :  Dict				    =    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
								return config, inputs_dict







@require_flax
class 						__snake_case (						lowerCAmelCase__				, unittest.TestCase  ):
		a_        =				True

		a_        =				(
		    (
		        FlaxElectraModel,
		        FlaxElectraForCausalLM,
		        FlaxElectraForMaskedLM,
		        FlaxElectraForPreTraining,
		        FlaxElectraForTokenClassification,
		        FlaxElectraForQuestionAnswering,
		        FlaxElectraForMultipleChoice,
		        FlaxElectraForSequenceClassification,
		    )
		    if is_flax_available()
		    else ()
		)




		def   _snake_case  (     self				:Optional[int])				-> List[Any]:






								'''simple docstring'''






								UpperCamelCase_    :  Optional[Any]				    =    FlaxElectraModelTester(self)





		@slow
		def   _snake_case  (     self				:Dict)				-> Optional[Any]:






								'''simple docstring'''






								for model_class_name in self.all_model_classes:
														if model_class_name == FlaxElectraForMaskedLM:
																				UpperCamelCase_    :  Any				    =    model_class_name.from_pretrained("""google/electra-small-generator""")
														else:
																				UpperCamelCase_    :  int				    =    model_class_name.from_pretrained("""google/electra-small-discriminator""")
														UpperCamelCase_    :  Optional[int]				    =    model(np.ones((1, 1)))
														self.assertIsNotNone(lowerCAmelCase_)


'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


__SCREAMING_SNAKE_CASE  =			logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE  =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__SCREAMING_SNAKE_CASE  =			{
    "vocab_file": {
        "microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json",
    },
}

__SCREAMING_SNAKE_CASE  =			{
    "microsoft/mpnet-base": 512,
}

__SCREAMING_SNAKE_CASE  =			{
    "microsoft/mpnet-base": {"do_lower_case": True},
}




class 						__snake_case (						_a  ):
		a_        =				VOCAB_FILES_NAMES
		a_        =				PRETRAINED_VOCAB_FILES_MAP
		a_        =				PRETRAINED_INIT_CONFIGURATION
		a_        =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a_        =				MPNetTokenizer
		a_        =				["input_ids", "attention_mask"]




		def __init__(     self				:Tuple ,						_SCREAMING_SNAKE_CASE				:Dict=None ,						_SCREAMING_SNAKE_CASE				:List[Any]=None ,						_SCREAMING_SNAKE_CASE				:Optional[Any]=True ,						_SCREAMING_SNAKE_CASE				:Any="<s>" ,						_SCREAMING_SNAKE_CASE				:int="</s>" ,						_SCREAMING_SNAKE_CASE				:Optional[Any]="</s>" ,						_SCREAMING_SNAKE_CASE				:Any="<s>" ,						_SCREAMING_SNAKE_CASE				:Optional[int]="[UNK]" ,						_SCREAMING_SNAKE_CASE				:List[Any]="<pad>" ,						_SCREAMING_SNAKE_CASE				:int="<mask>" ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=True ,						_SCREAMING_SNAKE_CASE				:Union[str, Any]=None ,						**_SCREAMING_SNAKE_CASE				:Optional[int] ,						)				-> Optional[Any]:






								'''simple docstring'''






								super().__init__(
								    _SCREAMING_SNAKE_CASE ,						tokenizer_file=_SCREAMING_SNAKE_CASE ,						do_lower_case=_SCREAMING_SNAKE_CASE ,						bos_token=_SCREAMING_SNAKE_CASE ,						eos_token=_SCREAMING_SNAKE_CASE ,						sep_token=_SCREAMING_SNAKE_CASE ,						cls_token=_SCREAMING_SNAKE_CASE ,						unk_token=_SCREAMING_SNAKE_CASE ,						pad_token=_SCREAMING_SNAKE_CASE ,						mask_token=_SCREAMING_SNAKE_CASE ,						tokenize_chinese_chars=_SCREAMING_SNAKE_CASE ,						strip_accents=_SCREAMING_SNAKE_CASE ,						**_SCREAMING_SNAKE_CASE ,						)

								UpperCamelCase_    :  Dict				    =    json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    pre_tok_state.get("""lowercase""" ,						_SCREAMING_SNAKE_CASE) != do_lower_case
								    or pre_tok_state.get("""strip_accents""" ,						_SCREAMING_SNAKE_CASE) != strip_accents
								):
														UpperCamelCase_    :  Any				    =    getattr(_SCREAMING_SNAKE_CASE ,						pre_tok_state.pop("""type"""))
														UpperCamelCase_    :  Optional[int]				    =    do_lower_case
														UpperCamelCase_    :  str				    =    strip_accents
														UpperCamelCase_    :  str				    =    pre_tok_class(**_SCREAMING_SNAKE_CASE)

								UpperCamelCase_    :  List[str]				    =    do_lower_case




		@property
		def   _snake_case  (     self				:Optional[int])				-> str:






								'''simple docstring'''






								if self._mask_token is None:
														if self.verbose:
																				logger.error("""Using mask_token, but it is not set yet.""")
														return None
								return str(self._mask_token)




		@mask_token.setter
		def   _snake_case  (     self				:Optional[Any] ,						_SCREAMING_SNAKE_CASE				:List[Any])				-> int:






								'''simple docstring'''






								UpperCamelCase_    :  Tuple				    =    AddedToken(_SCREAMING_SNAKE_CASE ,						lstrip=_SCREAMING_SNAKE_CASE ,						rstrip=_SCREAMING_SNAKE_CASE) if isinstance(_SCREAMING_SNAKE_CASE ,						_SCREAMING_SNAKE_CASE) else value
								UpperCamelCase_    :  int				    =    value




		def   _snake_case  (     self				:str ,						_SCREAMING_SNAKE_CASE				:int ,						_SCREAMING_SNAKE_CASE				:Any=None)				-> str:






								'''simple docstring'''






								UpperCamelCase_    :  Tuple				    =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
														return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




		def   _snake_case  (     self				:Optional[Any] ,						_SCREAMING_SNAKE_CASE				:List[int] ,						_SCREAMING_SNAKE_CASE				:Optional[List[int]] = None)				-> List[int]:






								'''simple docstring'''






								UpperCamelCase_    :  Optional[Any]				    =    [self.sep_token_id]
								UpperCamelCase_    :  Dict				    =    [self.cls_token_id]

								if token_ids_a is None:
														return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





		def   _snake_case  (     self				:List[str] ,						_SCREAMING_SNAKE_CASE				:str ,						_SCREAMING_SNAKE_CASE				:Optional[str] = None)				-> Tuple[str]:






								'''simple docstring'''






								UpperCamelCase_    :  List[str]				    =    self._tokenizer.model.save(_SCREAMING_SNAKE_CASE ,						name=_SCREAMING_SNAKE_CASE)
								return tuple(_SCREAMING_SNAKE_CASE)




