




'''simple docstring'''

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    RemBertConfig,
		    RemBertForCausalLM,
		    RemBertForMaskedLM,
		    RemBertForMultipleChoice,
		    RemBertForQuestionAnswering,
		    RemBertForSequenceClassification,
		    RemBertForTokenClassification,
		    RemBertModel,
		)
		from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class __magic_name__:






						def __init__(   self						:						str					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						Dict=1_3					,   __UpperCamelCase						:						List[str]=7					,   __UpperCamelCase						:						Optional[int]=True					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Tuple=True					,   __UpperCamelCase						:						Optional[Any]=True					,   __UpperCamelCase						:						str=9_9					,   __UpperCamelCase						:						str=3_2					,   __UpperCamelCase						:						str=1_8					,   __UpperCamelCase						:						List[str]=4_3					,   __UpperCamelCase						:						Optional[int]=5					,   __UpperCamelCase						:						List[str]=4					,   __UpperCamelCase						:						List[Any]=3_7					,   __UpperCamelCase						:						Optional[int]="gelu"					,   __UpperCamelCase						:						Any=0.1					,   __UpperCamelCase						:						List[str]=0.1					,   __UpperCamelCase						:						Dict=5_1_2					,   __UpperCamelCase						:						str=1_6					,   __UpperCamelCase						:						str=2					,   __UpperCamelCase						:						str=0.02					,   __UpperCamelCase						:						str=3					,   __UpperCamelCase						:						Tuple=4					,   __UpperCamelCase						:						Any=None					,   ):



									'''simple docstring'''






									snake_case__          =						parent
									snake_case__          =						batch_size
									snake_case__          =						seq_length
									snake_case__          =						is_training
									snake_case__          =						use_input_mask
									snake_case__          =						use_token_type_ids
									snake_case__          =						use_labels
									snake_case__          =						vocab_size
									snake_case__          =						hidden_size
									snake_case__          =						input_embedding_size
									snake_case__          =						output_embedding_size
									snake_case__          =						num_hidden_layers
									snake_case__          =						num_attention_heads
									snake_case__          =						intermediate_size
									snake_case__          =						hidden_act
									snake_case__          =						hidden_dropout_prob
									snake_case__          =						attention_probs_dropout_prob
									snake_case__          =						max_position_embeddings
									snake_case__          =						type_vocab_size
									snake_case__          =						type_sequence_label_size
									snake_case__          =						initializer_range
									snake_case__          =						num_labels
									snake_case__          =						num_choices
									snake_case__          =						scope






						def       __lowerCAmelCase(   self						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						ids_tensor([self.batch_size, self.seq_length]					,   self.vocab_size       )

									snake_case__          =						None
									if self.use_input_mask:
												snake_case__          =						random_attention_mask([self.batch_size, self.seq_length]       )

									snake_case__          =						None
									if self.use_token_type_ids:
												snake_case__          =						ids_tensor([self.batch_size, self.seq_length]					,   self.type_vocab_size       )

									snake_case__          =						None
									snake_case__          =						None
									snake_case__          =						None
									if self.use_labels:
												snake_case__          =						ids_tensor([self.batch_size]					,   self.type_sequence_label_size       )
												snake_case__          =						ids_tensor([self.batch_size, self.seq_length]					,   self.num_labels       )
												snake_case__          =						ids_tensor([self.batch_size]					,   self.num_choices       )

									snake_case__          =						RemBertConfig(
									    vocab_size=self.vocab_size					,   hidden_size=self.hidden_size					,   input_embedding_size=self.input_embedding_size					,   output_embedding_size=self.output_embedding_size					,   num_hidden_layers=self.num_hidden_layers					,   num_attention_heads=self.num_attention_heads					,   intermediate_size=self.intermediate_size					,   hidden_act=self.hidden_act					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   max_position_embeddings=self.max_position_embeddings					,   type_vocab_size=self.type_vocab_size					,   is_decoder=lowercase__					,   initializer_range=self.initializer_range					,   )

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									(
									    (
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			 
									)          =						self.prepare_config_and_inputs()

									snake_case__          =						True
									snake_case__          =						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
									snake_case__          =						ids_tensor([self.batch_size, self.seq_length]					,   vocab_size=2       )

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    encoder_hidden_states,
									    encoder_attention_mask,
									)






						def       __lowerCAmelCase(   self						:						Tuple					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						RemBertModel(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__       )
									snake_case__          =						model(lowercase__					,   token_type_ids=lowercase__       )
									snake_case__          =						model(lowercase__       )
									self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)       )






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Union[str, Any]					,   ):



									'''simple docstring'''






									snake_case__          =						True
									snake_case__          =						RemBertModel(lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   encoder_hidden_states=lowercase__					,   encoder_attention_mask=lowercase__					,   )
									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   encoder_hidden_states=lowercase__					,   )
									snake_case__          =						model(lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__       )
									self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)       )






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						List[Any]					,   ):



									'''simple docstring'''






									snake_case__          =						RemBertForCausalLM(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   labels=lowercase__       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)       )






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						Tuple					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						RemBertForMaskedLM(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   labels=lowercase__       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)       )






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						str					,   ):



									'''simple docstring'''






									snake_case__          =						True
									snake_case__          =						True
									snake_case__          =						RemBertForCausalLM(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()

									# first forward pass
									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   encoder_hidden_states=lowercase__					,   encoder_attention_mask=lowercase__					,   use_cache=lowercase__					,   )
									snake_case__          =						outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									snake_case__          =						ids_tensor((self.batch_size, 3)					,   config.vocab_size       )
									snake_case__          =						ids_tensor((self.batch_size, 3)					,   vocab_size=2       )

									# append to next input_ids and
									snake_case__          =						torch.cat([input_ids, next_tokens]					,   dim=-1       )
									snake_case__          =						torch.cat([input_mask, next_mask]					,   dim=-1       )

									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   encoder_hidden_states=lowercase__					,   encoder_attention_mask=lowercase__					,   output_hidden_states=lowercase__					,   )["""hidden_states"""][0]
									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   encoder_hidden_states=lowercase__					,   encoder_attention_mask=lowercase__					,   past_key_values=lowercase__					,   output_hidden_states=lowercase__					,   )["""hidden_states"""][0]

									# select random slice
									snake_case__          =						ids_tensor((1,)					,   output_from_past.shape[-1]       ).item()
									snake_case__          =						output_from_no_past[:, -3:, random_slice_idx].detach()
									snake_case__          =						output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(lowercase__					,   lowercase__					,   atol=1E-3       )       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						Tuple					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						RemBertForQuestionAnswering(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   start_positions=lowercase__					,   end_positions=lowercase__					,   )
									self.parent.assertEqual(result.start_logits.shape					,   (self.batch_size, self.seq_length)       )
									self.parent.assertEqual(result.end_logits.shape					,   (self.batch_size, self.seq_length)       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						Dict					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						self.num_labels
									snake_case__          =						RemBertForSequenceClassification(lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   labels=lowercase__       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.num_labels)       )






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						int					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.num_labels
									snake_case__          =						RemBertForTokenClassification(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						model(lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   labels=lowercase__       )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.num_labels)       )






						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						Union[str, Any]					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						self.num_choices
									snake_case__          =						RemBertForMultipleChoice(config=lowercase__       )
									model.to(lowercase__       )
									model.eval()
									snake_case__          =						input_ids.unsqueeze(1       ).expand(-1					,   self.num_choices					,   -1       ).contiguous()
									snake_case__          =						token_type_ids.unsqueeze(1       ).expand(-1					,   self.num_choices					,   -1       ).contiguous()
									snake_case__          =						input_mask.unsqueeze(1       ).expand(-1					,   self.num_choices					,   -1       ).contiguous()
									snake_case__          =						model(
									    lowercase__					,   attention_mask=lowercase__					,   token_type_ids=lowercase__					,   labels=lowercase__					,   )
									self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.num_choices)       )







						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						self.prepare_config_and_inputs()
									(
									    (
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			 
									)          =						config_and_inputs
									snake_case__          =						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
									return config, inputs_dict







@require_torch
class __magic_name__(						a__    ,		a__    ,		unittest.TestCase      ):
						UpperCAmelCase_		:					str    =  (
						    (
						        RemBertModel,
						        RemBertForMaskedLM,
						        RemBertForCausalLM,
						        RemBertForMultipleChoice,
						        RemBertForQuestionAnswering,
						        RemBertForSequenceClassification,
						        RemBertForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						UpperCAmelCase_		:					Union[str, Any]    =  (RemBertForCausalLM,) if is_torch_available() else ()
						UpperCAmelCase_		:					Tuple    =  (
						    {
						        """feature-extraction""": RemBertModel,
						        """fill-mask""": RemBertForMaskedLM,
						        """question-answering""": RemBertForQuestionAnswering,
						        """text-classification""": RemBertForSequenceClassification,
						        """text-generation""": RemBertForCausalLM,
						        """token-classification""": RemBertForTokenClassification,
						        """zero-shot""": RemBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__          =						RemBertModelTester(self       )
									snake_case__          =						ConfigTester(self					,   config_class=lowercase__					,   hidden_size=3_7       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									self.config_tester.run_common_tests()






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowercase__       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
												snake_case__          =						type
												self.model_tester.create_and_check_model(*lowercase__       )






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*lowercase__       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*lowercase__       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowercase__       )






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*lowercase__       )






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*lowercase__       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*lowercase__       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_model_as_decoder(*lowercase__       )






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									(
									    (
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			(
									    snake_case__ 
									)    ,			 
									)          =						self.model_tester.prepare_config_and_inputs_for_decoder()

									snake_case__          =						None

									self.model_tester.create_and_check_model_as_decoder(
									    lowercase__					,   lowercase__					,   lowercase__					,   lowercase__					,   lowercase__					,   lowercase__					,   lowercase__					,   lowercase__					,   lowercase__					,   )







						@slow
						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												snake_case__          =						RemBertModel.from_pretrained(lowercase__       )
												self.assertIsNotNone(lowercase__       )







@require_torch
class __magic_name__(						unittest.TestCase      ):






						@slow
						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						RemBertModel.from_pretrained("""google/rembert"""       )
									snake_case__          =						torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]       )
									snake_case__          =						torch.tensor([[0, 0, 0, 1, 1]]       )
									with torch.no_grad():
												snake_case__          =						model(lowercase__					,   token_type_ids=lowercase__					,   output_hidden_states=lowercase__       )

									snake_case__          =						1_1_5_2

									snake_case__          =						torch.Size((1, 5, hidden_size)       )
									self.assertEqual(output["""last_hidden_state"""].shape					,   lowercase__       )

									snake_case__          =						torch.tensor(
									    [
									        [
									            [0.07_54, -0.20_22, 0.19_04],
									            [-0.33_54, -0.36_92, -0.47_91],
									            [-0.23_14, -0.67_29, -0.07_49],
									            [-0.03_96, -0.31_05, -0.42_34],
									            [-0.15_71, -0.05_25, 0.53_53],
									        ]
									    ]       )

									# Running on the original tf implementation gives slightly different results here.
									# Not clear why this variations is present
									# TODO: Find reason for discrepancy
									# expected_original_implementation = [[
									#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
									#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
									#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
									#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
									#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
									# ]]

									self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3]					,   lowercase__					,   atol=1E-4       )       )





import unittest

from transformers import ElectraConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      ElectraForCausalLM,
      ElectraForMaskedLM,
      ElectraForMultipleChoice,
      ElectraForPreTraining,
      ElectraForQuestionAnswering,
      ElectraForSequenceClassification,
      ElectraForTokenClassification,
      ElectraModel,
  )
  from transformers.models.electra.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST
class 							__lowercase   :



     def __init__(  self   :							Optional[Any]      ,		lowercase__   :							int      ,		lowercase__   :							Tuple=1_3      ,		lowercase__   :							Any=7      ,		lowercase__   :							Optional[int]=True      ,		lowercase__   :							str=True      ,		lowercase__   :							Optional[Any]=True      ,		lowercase__   :							int=True      ,		lowercase__   :							str=9_9      ,		lowercase__   :							Dict=3_2      ,		lowercase__   :							Union[str, Any]=5      ,		lowercase__   :							Any=4      ,		lowercase__   :							Tuple=3_7      ,		lowercase__   :							Tuple="gelu"      ,		lowercase__   :							Optional[int]=0.1      ,		lowercase__   :							str=0.1      ,		lowercase__   :							int=5_1_2      ,		lowercase__   :							Any=1_6      ,		lowercase__   :							Dict=2      ,		lowercase__   :							Tuple=0.02      ,		lowercase__   :							Union[str, Any]=3      ,		lowercase__   :							Tuple=4      ,		lowercase__   :							Dict=None      ,		):
       a_		    =  parent
       a_		    =  batch_size
       a_		    =  seq_length
       a_		    =  is_training
       a_		    =  use_input_mask
       a_		    =  use_token_type_ids
       a_		    =  use_labels
       a_		    =  vocab_size
       a_		    =  hidden_size
       a_		    =  num_hidden_layers
       a_		    =  num_attention_heads
       a_		    =  intermediate_size
       a_		    =  hidden_act
       a_		    =  hidden_dropout_prob
       a_		    =  attention_probs_dropout_prob
       a_		    =  max_position_embeddings
       a_		    =  type_vocab_size
       a_		    =  type_sequence_label_size
       a_		    =  initializer_range
       a_		    =  num_labels
       a_		    =  num_choices
       a_		    =  scope



     def 				__magic_name__		(  self   :							str       ):
       a_		    =  ids_tensor([self.batch_size, self.seq_length]      ,		self.vocab_size       )

       a_		    =  None
       if self.use_input_mask:
         a_		    =  random_attention_mask([self.batch_size, self.seq_length]       )

       a_		    =  None
       if self.use_token_type_ids:
         a_		    =  ids_tensor([self.batch_size, self.seq_length]      ,		self.type_vocab_size       )

       a_		    =  None
       a_		    =  None
       a_		    =  None
       if self.use_labels:
         a_		    =  ids_tensor([self.batch_size]      ,		self.type_sequence_label_size       )
         a_		    =  ids_tensor([self.batch_size, self.seq_length]      ,		self.num_labels       )
         a_		    =  ids_tensor([self.batch_size]      ,		self.num_choices       )
         a_		    =  ids_tensor([self.batch_size, self.seq_length]      ,		1       )

       a_		    =  self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           fake_token_labels,
       )



     def 				__magic_name__		(  self   :							Any       ):
       return ElectraConfig(
           vocab_size=self.vocab_size      ,		hidden_size=self.hidden_size      ,		num_hidden_layers=self.num_hidden_layers      ,		num_attention_heads=self.num_attention_heads      ,		intermediate_size=self.intermediate_size      ,		hidden_act=self.hidden_act      ,		hidden_dropout_prob=self.hidden_dropout_prob      ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,		max_position_embeddings=self.max_position_embeddings      ,		type_vocab_size=self.type_vocab_size      ,		is_decoder=lowercase__      ,		initializer_range=self.initializer_range      ,		)



     def 				__magic_name__		(  self   :							Dict       ):
       (
           (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , 
       )		    =  self.prepare_config_and_inputs()

       a_		    =  True
       a_		    =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
       a_		    =  ids_tensor([self.batch_size, self.seq_length]      ,		vocab_size=2       )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )



     def 				__magic_name__		(  self   :							Union[str, Any]      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							int      ,		lowercase__   :							List[Any]      ,		lowercase__   :							List[Any]      ,		lowercase__   :							str      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							str      ,		lowercase__   :							str      ,		):
       a_		    =  ElectraModel(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__       )
       a_		    =  model(lowercase__      ,		token_type_ids=lowercase__       )
       a_		    =  model(lowercase__       )
       self.parent.assertEqual(result.last_hidden_state.shape      ,		(self.batch_size, self.seq_length, self.hidden_size)       )



     def 				__magic_name__		(  self   :							Optional[int]      ,		lowercase__   :							int      ,		lowercase__   :							Tuple      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							Any      ,		lowercase__   :							Dict      ,		lowercase__   :							Optional[Any]      ,		):
       a_		    =  True
       a_		    =  ElectraModel(lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(
           lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		encoder_hidden_states=lowercase__      ,		encoder_attention_mask=lowercase__      ,		)
       a_		    =  model(
           lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		encoder_hidden_states=lowercase__      ,		)
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__       )
       self.parent.assertEqual(result.last_hidden_state.shape      ,		(self.batch_size, self.seq_length, self.hidden_size)       )



     def 				__magic_name__		(  self   :							str      ,		lowercase__   :							List[Any]      ,		lowercase__   :							Any      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							Tuple      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							Any      ,		lowercase__   :							Dict      ,		lowercase__   :							Any      ,		):
       a_		    =  ElectraForMaskedLM(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		labels=lowercase__       )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.vocab_size)       )



     def 				__magic_name__		(  self   :							Optional[int]      ,		lowercase__   :							Dict      ,		lowercase__   :							Dict      ,		lowercase__   :							Dict      ,		lowercase__   :							str      ,		lowercase__   :							List[Any]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							int      ,		lowercase__   :							str      ,		):
       a_		    =  ElectraForCausalLM(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		labels=lowercase__       )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.vocab_size)       )



     def 				__magic_name__		(  self   :							str      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							List[Any]      ,		lowercase__   :							str      ,		lowercase__   :							List[str]      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							Dict      ,		lowercase__   :							Dict      ,		lowercase__   :							List[Any]      ,		):
       a_		    =  self.num_labels
       a_		    =  ElectraForTokenClassification(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		labels=lowercase__       )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.num_labels)       )



     def 				__magic_name__		(  self   :							Any      ,		lowercase__   :							Any      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Any      ,		lowercase__   :							Dict      ,		lowercase__   :							List[str]      ,		lowercase__   :							List[str]      ,		lowercase__   :							List[str]      ,		):
       a_		    =  self.num_labels
       a_		    =  ElectraForPreTraining(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		labels=lowercase__       )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length)       )



     def 				__magic_name__		(  self   :							Optional[int]      ,		lowercase__   :							Dict      ,		lowercase__   :							Tuple      ,		lowercase__   :							Any      ,		lowercase__   :							Any      ,		lowercase__   :							int      ,		lowercase__   :							List[str]      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							int      ,		):
       a_		    =  self.num_labels
       a_		    =  ElectraForSequenceClassification(lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		labels=lowercase__       )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_labels)       )



     def 				__magic_name__		(  self   :							Any      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							int      ,		lowercase__   :							Any      ,		lowercase__   :							List[Any]      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							List[str]      ,		lowercase__   :							Optional[Any]      ,		):
       a_		    =  ElectraForQuestionAnswering(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  model(
           lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		start_positions=lowercase__      ,		end_positions=lowercase__      ,		)
       self.parent.assertEqual(result.start_logits.shape      ,		(self.batch_size, self.seq_length)       )
       self.parent.assertEqual(result.end_logits.shape      ,		(self.batch_size, self.seq_length)       )



     def 				__magic_name__		(  self   :							int      ,		lowercase__   :							List[str]      ,		lowercase__   :							List[Any]      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							Tuple      ,		lowercase__   :							List[Any]      ,		lowercase__   :							List[Any]      ,		lowercase__   :							Dict      ,		lowercase__   :							Optional[Any]      ,		):
       a_		    =  self.num_choices
       a_		    =  ElectraForMultipleChoice(config=lowercase__       )
       model.to(lowercase__       )
       model.eval()
       a_		    =  input_ids.unsqueeze(1       ).expand(-1      ,		self.num_choices      ,		-1       ).contiguous()
       a_		    =  token_type_ids.unsqueeze(1       ).expand(-1      ,		self.num_choices      ,		-1       ).contiguous()
       a_		    =  input_mask.unsqueeze(1       ).expand(-1      ,		self.num_choices      ,		-1       ).contiguous()
       a_		    =  model(
           lowercase__      ,		attention_mask=lowercase__      ,		token_type_ids=lowercase__      ,		labels=lowercase__      ,		)
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_choices)       )



     def 				__magic_name__		(  self   :							Tuple       ):
       a_		    =  self.prepare_config_and_inputs()
       (
           (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , (
           a_
       )    , 
       )		    =  config_and_inputs
       a_		    =  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict



@require_torch
class 							__lowercase   (			a__	,		a__	,		unittest.TestCase    ):
     _lowerCAmelCase		       =						(
         (
             ElectraModel,
             ElectraForPreTraining,
             ElectraForMaskedLM,
             ElectraForCausalLM,
             ElectraForMultipleChoice,
             ElectraForTokenClassification,
             ElectraForSequenceClassification,
             ElectraForQuestionAnswering,
         )
         if is_torch_available()
         else ()
     )
     _lowerCAmelCase		       =						(
         {
             "feature-extraction": ElectraModel,
             "fill-mask": ElectraForMaskedLM,
             "question-answering": ElectraForQuestionAnswering,
             "text-classification": ElectraForSequenceClassification,
             "text-generation": ElectraForCausalLM,
             "token-classification": ElectraForTokenClassification,
             "zero-shot": ElectraForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     _lowerCAmelCase		       =						True



     def 				__magic_name__		(  self   :							Tuple      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							List[Any]      ,		lowercase__   :							Dict=False       ):
       a_		    =  super()._prepare_for_class(lowercase__      ,		lowercase__      ,		return_labels=lowercase__       )

       if return_labels:
         if model_class in get_values(lowercase__       ):
           a_		    =  torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)      ,		dtype=torch.long      ,		device=lowercase__       )
       return inputs_dict



     def 				__magic_name__		(  self   :							List[str]       ):
       a_		    =  ElectraModelTester(self       )
       a_		    =  ConfigTester(self      ,		config_class=lowercase__      ,		hidden_size=3_7       )



     def 				__magic_name__		(  self   :							str       ):
       self.config_tester.run_common_tests()



     def 				__magic_name__		(  self   :							Any       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_model(*lowercase__       )



     def 				__magic_name__		(  self   :							Optional[Any]       ):
       a_		    =  self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_electra_model_as_decoder(*lowercase__       )



     def 				__magic_name__		(  self   :							int       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
         a_		    =  type
         self.model_tester.create_and_check_electra_model(*lowercase__       )



     def 				__magic_name__		(  self   :							List[str]       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_masked_lm(*lowercase__       )



     def 				__magic_name__		(  self   :							Optional[int]       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_token_classification(*lowercase__       )



     def 				__magic_name__		(  self   :							Optional[Any]       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_pretraining(*lowercase__       )



     def 				__magic_name__		(  self   :							List[Any]       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_sequence_classification(*lowercase__       )



     def 				__magic_name__		(  self   :							List[str]       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_question_answering(*lowercase__       )



     def 				__magic_name__		(  self   :							str       ):
       a_		    =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_electra_for_multiple_choice(*lowercase__       )



     @slow
     def 				__magic_name__		(  self   :							Union[str, Any]       ):
       for model_name in ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         a_		    =  ElectraModel.from_pretrained(lowercase__       )
         self.assertIsNotNone(lowercase__       )



     def 				__magic_name__		(  self   :							Dict       ):
       a_		    =  self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_electra_for_causal_lm(*lowercase__       )



@require_torch
class 							__lowercase   (			unittest.TestCase    ):



     @slow
     def 				__magic_name__		(  self   :							Tuple       ):
       a_		    =  ElectraModel.from_pretrained('''google/electra-small-discriminator'''       )
       a_		    =  torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]       )
       a_		    =  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]       )
       a_		    =  model(lowercase__      ,		attention_mask=lowercase__       )[0]
       a_		    =  torch.Size((1, 1_1, 2_5_6)       )
       self.assertEqual(output.shape      ,		lowercase__       )
       a_		    =  torch.tensor(
           [[[0.4471, 0.6821, -0.3265], [0.4627, 0.5255, -0.3668], [0.4532, 0.3313, -0.4344]]]       )

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,		lowercase__      ,		atol=1e-4       )       )
