






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    RemBertConfig,
	    RemBertForCausalLM,
	    RemBertForMaskedLM,
	    RemBertForMultipleChoice,
	    RemBertForQuestionAnswering,
	    RemBertForSequenceClassification,
	    RemBertForTokenClassification,
	    RemBertModel,
	)
	from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class        __snake_case:



	def __init__(	self   :   Dict ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   int=13 ,		UpperCAmelCase__   :   int=7 ,		UpperCAmelCase__   :   Optional[int]=True ,		UpperCAmelCase__   :   Dict=True ,		UpperCAmelCase__   :   List[Any]=True ,		UpperCAmelCase__   :   Any=True ,		UpperCAmelCase__   :   Union[str, Any]=99 ,		UpperCAmelCase__   :   Tuple=32 ,		UpperCAmelCase__   :   str=18 ,		UpperCAmelCase__   :   Optional[int]=43 ,		UpperCAmelCase__   :   Optional[Any]=5 ,		UpperCAmelCase__   :   Optional[int]=4 ,		UpperCAmelCase__   :   str=37 ,		UpperCAmelCase__   :   Tuple="gelu" ,		UpperCAmelCase__   :   List[Any]=0.1 ,		UpperCAmelCase__   :   Union[str, Any]=0.1 ,		UpperCAmelCase__   :   str=512 ,		UpperCAmelCase__   :   List[str]=16 ,		UpperCAmelCase__   :   List[Any]=2 ,		UpperCAmelCase__   :   int=0.02 ,		UpperCAmelCase__   :   int=3 ,		UpperCAmelCase__   :   List[Any]=4 ,		UpperCAmelCase__   :   Dict=None ,		):

				"""simple docstring"""






				lowercase__						:	Dict         = parent
				lowercase__						:	Optional[Any]         = batch_size
				lowercase__						:	List[str]         = seq_length
				lowercase__						:	Optional[Any]         = is_training
				lowercase__						:	int         = use_input_mask
				lowercase__						:	Any         = use_token_type_ids
				lowercase__						:	Union[str, Any]         = use_labels
				lowercase__						:	List[str]         = vocab_size
				lowercase__						:	Any         = hidden_size
				lowercase__						:	Tuple         = input_embedding_size
				lowercase__						:	List[Any]         = output_embedding_size
				lowercase__						:	str         = num_hidden_layers
				lowercase__						:	Union[str, Any]         = num_attention_heads
				lowercase__						:	List[str]         = intermediate_size
				lowercase__						:	Tuple         = hidden_act
				lowercase__						:	List[Any]         = hidden_dropout_prob
				lowercase__						:	Optional[int]         = attention_probs_dropout_prob
				lowercase__						:	Union[str, Any]         = max_position_embeddings
				lowercase__						:	List[str]         = type_vocab_size
				lowercase__						:	str         = type_sequence_label_size
				lowercase__						:	Any         = initializer_range
				lowercase__						:	List[str]         = num_labels
				lowercase__						:	Union[str, Any]         = num_choices
				lowercase__						:	Optional[int]         = scope



	def 						UpperCAmelCase    (	self   :   Dict	):

				"""simple docstring"""






				lowercase__						:	List[Any]         = ids_tensor([self.batch_size, self.seq_length] ,		self.vocab_size	)

				lowercase__						:	Optional[int]         = None
				if self.use_input_mask:
							lowercase__						:	List[Any]         = random_attention_mask([self.batch_size, self.seq_length]	)

				lowercase__						:	Optional[Any]         = None
				if self.use_token_type_ids:
							lowercase__						:	Union[str, Any]         = ids_tensor([self.batch_size, self.seq_length] ,		self.type_vocab_size	)

				lowercase__						:	Dict         = None
				lowercase__						:	str         = None
				lowercase__						:	List[Any]         = None
				if self.use_labels:
							lowercase__						:	Tuple         = ids_tensor([self.batch_size] ,		self.type_sequence_label_size	)
							lowercase__						:	Optional[int]         = ids_tensor([self.batch_size, self.seq_length] ,		self.num_labels	)
							lowercase__						:	Any         = ids_tensor([self.batch_size] ,		self.num_choices	)

				lowercase__						:	str         = RemBertConfig(
				    vocab_size=self.vocab_size ,		hidden_size=self.hidden_size ,		input_embedding_size=self.input_embedding_size ,		output_embedding_size=self.output_embedding_size ,		num_hidden_layers=self.num_hidden_layers ,		num_attention_heads=self.num_attention_heads ,		intermediate_size=self.intermediate_size ,		hidden_act=self.hidden_act ,		hidden_dropout_prob=self.hidden_dropout_prob ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob ,		max_position_embeddings=self.max_position_embeddings ,		type_vocab_size=self.type_vocab_size ,		is_decoder=UpperCAmelCase__ ,		initializer_range=self.initializer_range ,		)

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



	def 						UpperCAmelCase    (	self   :   int	):

				"""simple docstring"""






				(
				    lowercase__					
				)						:	List[str]         = self.prepare_config_and_inputs()

				lowercase__						:	List[str]         = True
				lowercase__						:	Optional[int]         = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
				lowercase__						:	Optional[Any]         = ids_tensor([self.batch_size, self.seq_length] ,		vocab_size=2	)

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				    encoder_hidden_states,
				    encoder_attention_mask,
				)



	def 						UpperCAmelCase    (	self   :   Optional[Any] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   int	):

				"""simple docstring"""






				lowercase__						:	Any         = RemBertModel(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	Optional[int]         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__	)
				lowercase__						:	Optional[int]         = model(UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__	)
				lowercase__						:	Optional[Any]         = model(UpperCAmelCase__	)
				self.parent.assertEqual(result.last_hidden_state.shape ,		(self.batch_size, self.seq_length, self.hidden_size)	)



	def 						UpperCAmelCase    (	self   :   str ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Optional[Any] ,		):

				"""simple docstring"""






				lowercase__						:	List[Any]         = True
				lowercase__						:	Union[str, Any]         = RemBertModel(UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	int         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		)
				lowercase__						:	str         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		)
				lowercase__						:	Dict         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__	)
				self.parent.assertEqual(result.last_hidden_state.shape ,		(self.batch_size, self.seq_length, self.hidden_size)	)



	def 						UpperCAmelCase    (	self   :   List[Any] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Union[str, Any] ,		):

				"""simple docstring"""






				lowercase__						:	Optional[Any]         = RemBertForCausalLM(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	Optional[Any]         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
				self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.seq_length, self.vocab_size)	)



	def 						UpperCAmelCase    (	self   :   Any ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Any	):

				"""simple docstring"""






				lowercase__						:	Any         = RemBertForMaskedLM(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	int         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
				self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.seq_length, self.vocab_size)	)



	def 						UpperCAmelCase    (	self   :   List[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Union[str, Any] ,		):

				"""simple docstring"""






				lowercase__						:	Optional[int]         = True
				lowercase__						:	Optional[int]         = True
				lowercase__						:	int         = RemBertForCausalLM(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()

				# first forward pass
				lowercase__						:	List[str]         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		use_cache=UpperCAmelCase__ ,		)
				lowercase__						:	List[Any]         = outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				lowercase__						:	List[Any]         = ids_tensor((self.batch_size, 3) ,		config.vocab_size	)
				lowercase__						:	Union[str, Any]         = ids_tensor((self.batch_size, 3) ,		vocab_size=2	)

				# append to next input_ids and
				lowercase__						:	List[Any]         = torch.cat([input_ids, next_tokens] ,		dim=-1	)
				lowercase__						:	Dict         = torch.cat([input_mask, next_mask] ,		dim=-1	)

				lowercase__						:	Dict         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__ ,		)["""hidden_states"""][0]
				lowercase__						:	List[str]         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		past_key_values=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__ ,		)["""hidden_states"""][0]

				# select random slice
				lowercase__						:	Any         = ids_tensor((1,) ,		output_from_past.shape[-1]	).item()
				lowercase__						:	Tuple         = output_from_no_past[:, -3:, random_slice_idx].detach()
				lowercase__						:	Dict         = output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(UpperCAmelCase__ ,		UpperCAmelCase__ ,		atol=1e-3	)	)



	def 						UpperCAmelCase    (	self   :   Tuple ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   int	):

				"""simple docstring"""






				lowercase__						:	str         = RemBertForQuestionAnswering(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	Tuple         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		start_positions=UpperCAmelCase__ ,		end_positions=UpperCAmelCase__ ,		)
				self.parent.assertEqual(result.start_logits.shape ,		(self.batch_size, self.seq_length)	)
				self.parent.assertEqual(result.end_logits.shape ,		(self.batch_size, self.seq_length)	)



	def 						UpperCAmelCase    (	self   :   List[str] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Dict	):

				"""simple docstring"""






				lowercase__						:	Tuple         = self.num_labels
				lowercase__						:	List[str]         = RemBertForSequenceClassification(UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	str         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
				self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.num_labels)	)



	def 						UpperCAmelCase    (	self   :   Optional[int] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Dict	):

				"""simple docstring"""






				lowercase__						:	Dict         = self.num_labels
				lowercase__						:	Dict         = RemBertForTokenClassification(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	Any         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
				self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.seq_length, self.num_labels)	)



	def 						UpperCAmelCase    (	self   :   List[str] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   List[str]	):

				"""simple docstring"""






				lowercase__						:	Union[str, Any]         = self.num_choices
				lowercase__						:	Any         = RemBertForMultipleChoice(config=UpperCAmelCase__	)
				model.to(UpperCAmelCase__	)
				model.eval()
				lowercase__						:	Tuple         = input_ids.unsqueeze(1	).expand(-1 ,		self.num_choices ,		-1	).contiguous()
				lowercase__						:	Any         = token_type_ids.unsqueeze(1	).expand(-1 ,		self.num_choices ,		-1	).contiguous()
				lowercase__						:	Any         = input_mask.unsqueeze(1	).expand(-1 ,		self.num_choices ,		-1	).contiguous()
				lowercase__						:	Dict         = model(
				    UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__ ,		)
				self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.num_choices)	)







	def 						UpperCAmelCase    (	self   :   List[str]	):

				"""simple docstring"""






				lowercase__						:	List[Any]         = self.prepare_config_and_inputs()
				(
				    lowercase__					
				)						:	Union[str, Any]         = config_and_inputs
				lowercase__						:	Any         = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
				return config, inputs_dict
@require_torch
class        __snake_case(					__UpperCamelCase     ,       __UpperCamelCase     ,       unittest.TestCase      ):
	lowercase :       Tuple		      =		(
	    (
	        RemBertModel,
	        RemBertForMaskedLM,
	        RemBertForCausalLM,
	        RemBertForMultipleChoice,
	        RemBertForQuestionAnswering,
	        RemBertForSequenceClassification,
	        RemBertForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	lowercase :       Union[str, Any]		      =		(RemBertForCausalLM,) if is_torch_available() else ()
	lowercase :       Optional[int]		      =		(
	    {
	        '''feature-extraction''': RemBertModel,
	        '''fill-mask''': RemBertForMaskedLM,
	        '''question-answering''': RemBertForQuestionAnswering,
	        '''text-classification''': RemBertForSequenceClassification,
	        '''text-generation''': RemBertForCausalLM,
	        '''token-classification''': RemBertForTokenClassification,
	        '''zero-shot''': RemBertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)



	def 						UpperCAmelCase    (	self   :   Dict	):

				"""simple docstring"""






				lowercase__						:	List[Any]         = RemBertModelTester(self	)
				lowercase__						:	Any         = ConfigTester(self ,		config_class=UpperCAmelCase__ ,		hidden_size=37	)



	def 						UpperCAmelCase    (	self   :   Optional[int]	):

				"""simple docstring"""






				self.config_tester.run_common_tests()



	def 						UpperCAmelCase    (	self   :   Any	):

				"""simple docstring"""






				lowercase__						:	Any         = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   Dict	):

				"""simple docstring"""






				lowercase__						:	List[str]         = self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
							lowercase__						:	str         = type
							self.model_tester.create_and_check_model(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   str	):

				"""simple docstring"""






				lowercase__						:	Tuple         = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   int	):

				"""simple docstring"""






				lowercase__						:	Tuple         = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   Union[str, Any]	):

				"""simple docstring"""






				lowercase__						:	Tuple         = self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   int	):

				"""simple docstring"""






				lowercase__						:	Optional[Any]         = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   int	):

				"""simple docstring"""






				lowercase__						:	Tuple         = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   List[Any]	):

				"""simple docstring"""






				lowercase__						:	Optional[int]         = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   Any	):

				"""simple docstring"""






				lowercase__						:	List[Any]         = self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase__	)



	def 						UpperCAmelCase    (	self   :   List[Any]	):

				"""simple docstring"""






				(
				    lowercase__					
				)						:	Optional[int]         = self.model_tester.prepare_config_and_inputs_for_decoder()

				lowercase__						:	Dict         = None

				self.model_tester.create_and_check_model_as_decoder(
				    UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		)







	@slow
	def 						UpperCAmelCase    (	self   :   List[Any]	):

				"""simple docstring"""






				for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							lowercase__						:	Optional[int]         = RemBertModel.from_pretrained(UpperCAmelCase__	)
							self.assertIsNotNone(UpperCAmelCase__	)
@require_torch
class        __snake_case(					unittest.TestCase      ):



	@slow
	def 						UpperCAmelCase    (	self   :   List[str]	):

				"""simple docstring"""






				lowercase__						:	Tuple         = RemBertModel.from_pretrained("""google/rembert"""	)
				lowercase__						:	List[str]         = torch.tensor([[312, 56_498, 313, 2_125, 313]]	)
				lowercase__						:	Optional[Any]         = torch.tensor([[0, 0, 0, 1, 1]]	)
				with torch.no_grad():
							lowercase__						:	Dict         = model(UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__	)

				lowercase__						:	Union[str, Any]         = 1_152

				lowercase__						:	List[str]         = torch.Size((1, 5, hidden_size)	)
				self.assertEqual(output["""last_hidden_state"""].shape ,		UpperCAmelCase__	)

				lowercase__						:	List[Any]         = torch.tensor(
				    [
				        [
				            [0.0_754, -0.2_022, 0.1_904],
				            [-0.3_354, -0.3_692, -0.4_791],
				            [-0.2_314, -0.6_729, -0.0_749],
				            [-0.0_396, -0.3_105, -0.4_234],
				            [-0.1_571, -0.0_525, 0.5_353],
				        ]
				    ]	)

				# Running on the original tf implementation gives slightly different results here.
				# Not clear why this variations is present
				# TODO: Find reason for discrepancy
				# expected_original_implementation = [[
				#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
				#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
				#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
				#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
				#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
				# ]]

				self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3] ,		UpperCAmelCase__ ,		atol=1e-4	)	)










from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a    =       logging.get_logger(__name__)

a    =       {
    'nghuyong/ernie-1.0-base-zh': 'https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json',
    'nghuyong/ernie-2.0-base-en': 'https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json',
    'nghuyong/ernie-2.0-large-en': 'https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json',
    'nghuyong/ernie-3.0-base-zh': 'https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-medium-zh': 'https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-mini-zh': 'https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-micro-zh': 'https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-nano-zh': 'https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json',
    'nghuyong/ernie-gram-zh': 'https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json',
    'nghuyong/ernie-health-zh': 'https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json',
}
class        __snake_case(					__UpperCamelCase      ):
 lowercase :       Dict		      =		'''ernie'''



 def __init__(	self   :   Tuple ,		UpperCAmelCase__   :   int=30_522 ,		UpperCAmelCase__   :   List[str]=768 ,		UpperCAmelCase__   :   Tuple=12 ,		UpperCAmelCase__   :   Optional[int]=12 ,		UpperCAmelCase__   :   Any=3_072 ,		UpperCAmelCase__   :   Tuple="gelu" ,		UpperCAmelCase__   :   Union[str, Any]=0.1 ,		UpperCAmelCase__   :   str=0.1 ,		UpperCAmelCase__   :   List[str]=512 ,		UpperCAmelCase__   :   str=2 ,		UpperCAmelCase__   :   Optional[int]=3 ,		UpperCAmelCase__   :   Dict=False ,		UpperCAmelCase__   :   Any=0.02 ,		UpperCAmelCase__   :   Any=1e-12 ,		UpperCAmelCase__   :   str=0 ,		UpperCAmelCase__   :   Any="absolute" ,		UpperCAmelCase__   :   List[str]=True ,		UpperCAmelCase__   :   Tuple=None ,		**UpperCAmelCase__   :   Dict ,		):

    """simple docstring"""






    super().__init__(pad_token_id=UpperCAmelCase__ ,		**UpperCAmelCase__	)

    lowercase__						:	List[str]         = vocab_size
    lowercase__						:	Optional[int]         = hidden_size
    lowercase__						:	Union[str, Any]         = num_hidden_layers
    lowercase__						:	Any         = num_attention_heads
    lowercase__						:	Optional[int]         = hidden_act
    lowercase__						:	str         = intermediate_size
    lowercase__						:	Tuple         = hidden_dropout_prob
    lowercase__						:	Tuple         = attention_probs_dropout_prob
    lowercase__						:	str         = max_position_embeddings
    lowercase__						:	List[Any]         = type_vocab_size
    lowercase__						:	Any         = task_type_vocab_size
    lowercase__						:	Optional[Any]         = use_task_id
    lowercase__						:	Optional[Any]         = initializer_range
    lowercase__						:	Union[str, Any]         = layer_norm_eps
    lowercase__						:	Dict         = position_embedding_type
    lowercase__						:	Union[str, Any]         = use_cache
    lowercase__						:	int         = classifier_dropout
class        __snake_case(					__UpperCamelCase      ):



 @property
 def 						UpperCAmelCase    (	self   :   List[Any]	):

    """simple docstring"""






    if self.task == "multiple-choice":
       lowercase__						:	Dict         = {0: """batch""", 1: """choice""", 2: """sequence"""}
    else:
       lowercase__						:	Dict         = {0: """batch""", 1: """sequence"""}
    return OrderedDict(
        [
            ("""input_ids""", dynamic_axis),
            ("""attention_mask""", dynamic_axis),
            ("""token_type_ids""", dynamic_axis),
            ("""task_type_ids""", dynamic_axis),
        ]	)


