code: 
  '''simple docstring'''
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  						import torch
  
  						from transformers import (
  						    RemBertConfig,
  						    RemBertForCausalLM,
  						    RemBertForMaskedLM,
  						    RemBertForMultipleChoice,
  						    RemBertForQuestionAnswering,
  						    RemBertForSequenceClassification,
  						    RemBertForTokenClassification,
  						    RemBertModel,
  						)
  						from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class   snake_case__					:
  
  
  
  
  
  
  
  		"""simple docstring"""
  
  		def __init__(			self							: List[str]     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: List[str]=13     ,  UpperCamelCase__							: Tuple=7     ,  UpperCamelCase__							: int=True     ,  UpperCamelCase__							: Dict=True     ,  UpperCamelCase__							: List[Any]=True     ,  UpperCamelCase__							: int=True     ,  UpperCamelCase__							: Dict=99     ,  UpperCamelCase__							: Union[str, Any]=32     ,  UpperCamelCase__							: Union[str, Any]=18     ,  UpperCamelCase__							: Dict=43     ,  UpperCamelCase__							: List[str]=5     ,  UpperCamelCase__							: Union[str, Any]=4     ,  UpperCamelCase__							: Optional[Any]=37     ,  UpperCamelCase__							: int="gelu"     ,  UpperCamelCase__							: List[str]=0.1     ,  UpperCamelCase__							: int=0.1     ,  UpperCamelCase__							: Optional[int]=512     ,  UpperCamelCase__							: Optional[int]=16     ,  UpperCamelCase__							: Optional[int]=2     ,  UpperCamelCase__							: Any=0.02     ,  UpperCamelCase__							: Union[str, Any]=3     ,  UpperCamelCase__							: int=4     ,  UpperCamelCase__							: Optional[int]=None     ,  )	-> Dict:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Dict     =	parent
  				snake_case     :					Dict     =	batch_size
  				snake_case     :					Optional[Any]     =	seq_length
  				snake_case     :					List[Any]     =	is_training
  				snake_case     :					Any     =	use_input_mask
  				snake_case     :					Optional[Any]     =	use_token_type_ids
  				snake_case     :					List[Any]     =	use_labels
  				snake_case     :					List[Any]     =	vocab_size
  				snake_case     :					Any     =	hidden_size
  				snake_case     :					int     =	input_embedding_size
  				snake_case     :					Union[str, Any]     =	output_embedding_size
  				snake_case     :					Optional[int]     =	num_hidden_layers
  				snake_case     :					str     =	num_attention_heads
  				snake_case     :					Union[str, Any]     =	intermediate_size
  				snake_case     :					str     =	hidden_act
  				snake_case     :					str     =	hidden_dropout_prob
  				snake_case     :					Optional[int]     =	attention_probs_dropout_prob
  				snake_case     :					Optional[int]     =	max_position_embeddings
  				snake_case     :					List[str]     =	type_vocab_size
  				snake_case     :					Dict     =	type_sequence_label_size
  				snake_case     :					int     =	initializer_range
  				snake_case     :					int     =	num_labels
  				snake_case     :					Optional[int]     =	num_choices
  				snake_case     :					Union[str, Any]     =	scope
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[int]			)	-> Any:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Optional[int]     =	ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size			)
  
  				snake_case     :					Optional[int]     =	None
  				if self.use_input_mask:
  						snake_case     :					Optional[Any]     =	random_attention_mask([self.batch_size, self.seq_length]			)
  
  				snake_case     :					Dict     =	None
  				if self.use_token_type_ids:
  						snake_case     :					Any     =	ids_tensor([self.batch_size, self.seq_length]     ,  self.type_vocab_size			)
  
  				snake_case     :					Any     =	None
  				snake_case     :					str     =	None
  				snake_case     :					Optional[int]     =	None
  				if self.use_labels:
  						snake_case     :					List[Any]     =	ids_tensor([self.batch_size]     ,  self.type_sequence_label_size			)
  						snake_case     :					Optional[int]     =	ids_tensor([self.batch_size, self.seq_length]     ,  self.num_labels			)
  						snake_case     :					Optional[Any]     =	ids_tensor([self.batch_size]     ,  self.num_choices			)
  
  				snake_case     :					int     =	RemBertConfig(
  				    vocab_size=self.vocab_size     ,  hidden_size=self.hidden_size     ,  input_embedding_size=self.input_embedding_size     ,  output_embedding_size=self.output_embedding_size     ,  num_hidden_layers=self.num_hidden_layers     ,  num_attention_heads=self.num_attention_heads     ,  intermediate_size=self.intermediate_size     ,  hidden_act=self.hidden_act     ,  hidden_dropout_prob=self.hidden_dropout_prob     ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,  max_position_embeddings=self.max_position_embeddings     ,  type_vocab_size=self.type_vocab_size     ,  is_decoder=__lowerCamelCase     ,  initializer_range=self.initializer_range     ,  )
  
  				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  		def 			lowerCAmelCase							(			self							: Dict			)	-> Optional[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				(
  				    snake_case    
  				)     :					Optional[int]     =	self.prepare_config_and_inputs()
  
  				snake_case     :					Optional[int]     =	True
  				snake_case     :					List[Any]     =	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
  				snake_case     :					Any     =	ids_tensor([self.batch_size, self.seq_length]     ,  vocab_size=2			)
  
  				return (
  				    config,
  				    input_ids,
  				    token_type_ids,
  				    input_mask,
  				    sequence_labels,
  				    token_labels,
  				    choice_labels,
  				    encoder_hidden_states,
  				    encoder_attention_mask,
  				)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[Any]     ,  UpperCamelCase__							: List[Any]     ,  UpperCamelCase__							: Optional[Any]     ,  UpperCamelCase__							: int     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: Dict			)	-> Tuple:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Optional[int]     =	RemBertModel(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					int     =	model(__lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase			)
  				snake_case     :					Optional[int]     =	model(__lowerCamelCase     ,  token_type_ids=__lowerCamelCase			)
  				snake_case     :					Any     =	model(__lowerCamelCase			)
  				self.parent.assertEqual(result.last_hidden_state.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Dict     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: str     ,  UpperCamelCase__							: int     ,  UpperCamelCase__							: int     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: Optional[Any]     ,  )	-> Optional[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					List[Any]     =	True
  				snake_case     :					List[Any]     =	RemBertModel(__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					Any     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  encoder_hidden_states=__lowerCamelCase     ,  encoder_attention_mask=__lowerCamelCase     ,  )
  				snake_case     :					List[str]     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  encoder_hidden_states=__lowerCamelCase     ,  )
  				snake_case     :					Optional[int]     =	model(__lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase			)
  				self.parent.assertEqual(result.last_hidden_state.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: List[Any]     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: Dict     ,  UpperCamelCase__							: Optional[Any]     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: List[Any]     ,  )	-> int:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					int     =	RemBertForCausalLM(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					str     =	model(__lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  labels=__lowerCamelCase			)
  				self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[int]     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: Optional[Any]     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: int     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: Dict     ,  UpperCamelCase__							: Optional[Any]			)	-> str:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					str     =	RemBertForMaskedLM(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					Dict     =	model(__lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  labels=__lowerCamelCase			)
  				self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: List[Any]     ,  UpperCamelCase__							: List[Any]     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: int     ,  UpperCamelCase__							: List[Any]     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: Dict     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: int     ,  )	-> str:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Optional[int]     =	True
  				snake_case     :					int     =	True
  				snake_case     :					int     =	RemBertForCausalLM(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  
  				# first forward pass
  				snake_case     :					Any     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  encoder_hidden_states=__lowerCamelCase     ,  encoder_attention_mask=__lowerCamelCase     ,  use_cache=__lowerCamelCase     ,  )
  				snake_case     :					int     =	outputs.past_key_values
  
  				# create hypothetical multiple next token and extent to next_input_ids
  				snake_case     :					Any     =	ids_tensor((self.batch_size, 3)     ,  config.vocab_size			)
  				snake_case     :					Optional[int]     =	ids_tensor((self.batch_size, 3)     ,  vocab_size=2			)
  
  				# append to next input_ids and
  				snake_case     :					str     =	torch.cat([input_ids, next_tokens]     ,  dim=-1			)
  				snake_case     :					Dict     =	torch.cat([input_mask, next_mask]     ,  dim=-1			)
  
  				snake_case     :					Any     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  encoder_hidden_states=__lowerCamelCase     ,  encoder_attention_mask=__lowerCamelCase     ,  output_hidden_states=__lowerCamelCase     ,  )["hidden_states"][0]
  				snake_case     :					Dict     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  encoder_hidden_states=__lowerCamelCase     ,  encoder_attention_mask=__lowerCamelCase     ,  past_key_values=__lowerCamelCase     ,  output_hidden_states=__lowerCamelCase     ,  )["hidden_states"][0]
  
  				# select random slice
  				snake_case     :					Any     =	ids_tensor((1,)     ,  output_from_past.shape[-1]			).item()
  				snake_case     :					Optional[int]     =	output_from_no_past[:, -3:, random_slice_idx].detach()
  				snake_case     :					List[str]     =	output_from_past[:, :, random_slice_idx].detach()
  
  				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)
  
  				# test that outputs are equal for slice
  				self.parent.assertTrue(torch.allclose(__lowerCamelCase     ,  __lowerCamelCase     ,  atol=1e-3			)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Union[str, Any]     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: str     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: str     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: List[str]			)	-> List[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					List[str]     =	RemBertForQuestionAnswering(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					Tuple     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  start_positions=__lowerCamelCase     ,  end_positions=__lowerCamelCase     ,  )
  				self.parent.assertEqual(result.start_logits.shape     ,  (self.batch_size, self.seq_length)			)
  				self.parent.assertEqual(result.end_logits.shape     ,  (self.batch_size, self.seq_length)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: int     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: int     ,  UpperCamelCase__							: Optional[int]     ,  UpperCamelCase__							: Dict     ,  UpperCamelCase__							: Optional[Any]     ,  UpperCamelCase__							: Optional[int]			)	-> List[str]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Union[str, Any]     =	self.num_labels
  				snake_case     :					str     =	RemBertForSequenceClassification(__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					int     =	model(__lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  labels=__lowerCamelCase			)
  				self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_labels)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: List[str]     ,  UpperCamelCase__							: str     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: int			)	-> Optional[int]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					str     =	self.num_labels
  				snake_case     :					Tuple     =	RemBertForTokenClassification(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					Optional[int]     =	model(__lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  labels=__lowerCamelCase			)
  				self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.num_labels)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: str     ,  UpperCamelCase__							: List[Any]     ,  UpperCamelCase__							: List[str]     ,  UpperCamelCase__							: Dict     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: Union[str, Any]     ,  UpperCamelCase__							: List[Any]			)	-> Any:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Optional[int]     =	self.num_choices
  				snake_case     :					Tuple     =	RemBertForMultipleChoice(config=__lowerCamelCase			)
  				model.to(__lowerCamelCase			)
  				model.eval()
  				snake_case     :					str     =	input_ids.unsqueeze(1			).expand(-1     ,  self.num_choices     ,  -1			).contiguous()
  				snake_case     :					List[str]     =	token_type_ids.unsqueeze(1			).expand(-1     ,  self.num_choices     ,  -1			).contiguous()
  				snake_case     :					List[str]     =	input_mask.unsqueeze(1			).expand(-1     ,  self.num_choices     ,  -1			).contiguous()
  				snake_case     :					Optional[Any]     =	model(
  				    __lowerCamelCase     ,  attention_mask=__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  labels=__lowerCamelCase     ,  )
  				self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_choices)			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Any			)	-> int:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Union[str, Any]     =	self.prepare_config_and_inputs()
  				(
  				    snake_case    
  				)     :					Optional[Any]     =	config_and_inputs
  				snake_case     :					List[Any]     =	{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
  				return config, inputs_dict
  
  
  
  
  
  
  
  @require_torch
  class   snake_case__					(			__SCREAMING_SNAKE_CASE						,				__SCREAMING_SNAKE_CASE						,				unittest.TestCase				):
  
  
  
  
  
  
  
  		"""simple docstring"""
  
  		lowerCamelCase           =	(
  		    (
  		        RemBertModel,
  		        RemBertForMaskedLM,
  		        RemBertForCausalLM,
  		        RemBertForMultipleChoice,
  		        RemBertForQuestionAnswering,
  		        RemBertForSequenceClassification,
  		        RemBertForTokenClassification,
  		    )
  		    if is_torch_available()
  		    else ()
  		)
  		lowerCamelCase           =	(RemBertForCausalLM,) if is_torch_available() else ()
  		lowerCamelCase           =	(
  		    {
  		        """feature-extraction""": RemBertModel,
  		        """fill-mask""": RemBertForMaskedLM,
  		        """question-answering""": RemBertForQuestionAnswering,
  		        """text-classification""": RemBertForSequenceClassification,
  		        """text-generation""": RemBertForCausalLM,
  		        """token-classification""": RemBertForTokenClassification,
  		        """zero-shot""": RemBertForSequenceClassification,
  		    }
  		    if is_torch_available()
  		    else {}
  		)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[Any]			)	-> Any:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Any     =	RemBertModelTester(self			)
  				snake_case     :					Optional[int]     =	ConfigTester(self     ,  config_class=__lowerCamelCase     ,  hidden_size=37			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[int]			)	-> Tuple:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				self.config_tester.run_common_tests()
  
  
  
  		def 			lowerCAmelCase							(			self							: int			)	-> List[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Any     =	self.model_tester.prepare_config_and_inputs()
  				self.model_tester.create_and_check_model(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Tuple			)	-> Tuple:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					List[Any]     =	self.model_tester.prepare_config_and_inputs()
  				for type in ["absolute", "relative_key", "relative_key_query"]:
  						snake_case     :					int     =	type
  						self.model_tester.create_and_check_model(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[int]			)	-> List[str]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Any     =	self.model_tester.prepare_config_and_inputs()
  				self.model_tester.create_and_check_for_masked_lm(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[int]			)	-> Union[str, Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Union[str, Any]     =	self.model_tester.prepare_config_and_inputs()
  				self.model_tester.create_and_check_for_multiple_choice(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Tuple			)	-> Optional[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					List[str]     =	self.model_tester.prepare_config_and_inputs_for_decoder()
  				self.model_tester.create_and_check_decoder_model_past_large_inputs(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Optional[int]			)	-> Optional[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Optional[Any]     =	self.model_tester.prepare_config_and_inputs()
  				self.model_tester.create_and_check_for_question_answering(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: List[Any]			)	-> List[Any]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Any     =	self.model_tester.prepare_config_and_inputs()
  				self.model_tester.create_and_check_for_sequence_classification(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: List[str]			)	-> int:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					Optional[int]     =	self.model_tester.prepare_config_and_inputs()
  				self.model_tester.create_and_check_for_token_classification(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: List[str]			)	-> Dict:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					List[Any]     =	self.model_tester.prepare_config_and_inputs_for_decoder()
  				self.model_tester.create_and_check_model_as_decoder(*__lowerCamelCase			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Tuple			)	-> int:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				(
  				    snake_case    
  				)     :					Union[str, Any]     =	self.model_tester.prepare_config_and_inputs_for_decoder()
  
  				snake_case     :					List[Any]     =	None
  
  				self.model_tester.create_and_check_model_as_decoder(
  				    __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  __lowerCamelCase     ,  )
  
  
  
  		@slow
  		def 			lowerCAmelCase							(			self							: List[Any]			)	-> Any:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  						snake_case     :					int     =	RemBertModel.from_pretrained(__lowerCamelCase			)
  						self.assertIsNotNone(__lowerCamelCase			)
  
  
  
  
  
  
  
  @require_torch
  class   snake_case__					(			unittest.TestCase				):
  
  
  
  
  
  
  
  		"""simple docstring"""
  
  		@slow
  		def 			lowerCAmelCase							(			self							: Dict			)	-> List[str]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					str     =	RemBertModel.from_pretrained('''google/rembert'''			)
  				snake_case     :					List[str]     =	torch.tensor([[312, 5_6498, 313, 2125, 313]]			)
  				snake_case     :					List[Any]     =	torch.tensor([[0, 0, 0, 1, 1]]			)
  				with torch.no_grad():
  						snake_case     :					Union[str, Any]     =	model(__lowerCamelCase     ,  token_type_ids=__lowerCamelCase     ,  output_hidden_states=__lowerCamelCase			)
  
  				snake_case     :					Union[str, Any]     =	1152
  
  				snake_case     :					Dict     =	torch.Size((1, 5, hidden_size)			)
  				self.assertEqual(output['''last_hidden_state'''].shape     ,  __lowerCamelCase			)
  
  				snake_case     :					Dict     =	torch.tensor(
  				    [
  				        [
  				            [0.0_754, -0.2_022, 0.1_904],
  				            [-0.3_354, -0.3_692, -0.4_791],
  				            [-0.2_314, -0.6_729, -0.0_749],
  				            [-0.0_396, -0.3_105, -0.4_234],
  				            [-0.1_571, -0.0_525, 0.5_353],
  				        ]
  				    ]			)
  
  				# Running on the original tf implementation gives slightly different results here.
  				# Not clear why this variations is present
  				# TODO: Find reason for discrepancy
  				# expected_original_implementation = [[
  				#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
  				#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
  				#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
  				#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
  				#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
  				# ]]
  
  				self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]     ,  __lowerCamelCase     ,  atol=1e-4			)			)
  
  
  
style_context: 
  '''simple docstring'''
  import torch
  
  from transformers import AutoTokenizer
  
  
  
  
  
  
  class   snake_case__					(			__SCREAMING_SNAKE_CASE				):
  
  
  
  
  
  
  
  		"""simple docstring"""
  
  		def __init__(			self							: Union[str, Any]     ,  UpperCamelCase__							: List[str]			)	-> List[str]:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					List[Any]     =	AutoTokenizer.from_pretrained(UpperCamelCase__			)
  
  
  
  		def 			lowerCAmelCase							(			self							: Any     ,  UpperCamelCase__							: str			)	-> Dict:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				if isinstance(UpperCamelCase__     ,  UpperCamelCase__			) and all(isinstance(_x     ,  UpperCamelCase__			) for _x in x			):
  						snake_case     :					Any     =	None
  						for l in x:
  								snake_case     :					str     =	self.tokenizer(
  								    UpperCamelCase__     ,  padding='''max_length'''     ,  max_length=384     ,  truncation=UpperCamelCase__     ,  return_tensors='''pt'''     ,  )
  								snake_case     :					Tuple     =	torch.tensor([len(UpperCamelCase__			)]			)
  								if d is not None:
  										for k in d.keys():
  												snake_case     :					int     =	torch.cat((d[k], t[k])     ,  0			)
  								else:
  										snake_case     :					List[str]     =	t
  
  						snake_case     :					Union[str, Any]     =	torch.tensor(self.tokenizer.convert_tokens_to_ids('''[E]'''			)			)
  						snake_case     :					int     =	torch.tensor(self.tokenizer.convert_tokens_to_ids('''[/E]'''			)			)
  
  				elif isinstance(UpperCamelCase__     ,  UpperCamelCase__			) and all(isinstance(_x     ,  UpperCamelCase__			) for _x in x			):
  						snake_case     :					Optional[int]     =	self.tokenizer(
  						    UpperCamelCase__     ,  padding='''max_length'''     ,  max_length=384     ,  truncation=UpperCamelCase__     ,  return_tensors='''pt'''     ,  )
  
  				else:
  						raise Exception(
  						    '''Type of parameter x was not recognized! Only `list of strings` for query or `list of lists of'''
  						    ''' strings` for supports are supported.'''			)
  
  				return d
  
  
  
  		def 			lowerCAmelCase							(			self							: int     ,  UpperCamelCase__							: List[Any]     ,  UpperCamelCase__							: List[Any]     ,  UpperCamelCase__							: Any     ,  UpperCamelCase__							: Tuple     ,  UpperCamelCase__							: Any=0.70			)	-> Dict:
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  
  				snake_case     :					str     =	[]
  				for idx in range(len(W_query['''input_ids''']			)			):
  						snake_case     :					Optional[int]     =	range(p_start.shape[1]			)
  
  						snake_case     :					str     =	[]
  						for start_id in start_indexes:
  								for end_id in end_indexes:
  										if start_id < end_id:
  												output.append(
  												    (
  												        start_id,
  												        end_id,
  												        p_start[idx][start_id].item(),
  												        p_end[idx][end_id].item(),
  												    )			)
  
  						output.sort(key=lambda UpperCamelCase__			: (tup[2] * tup[3])     ,  reverse=UpperCamelCase__			)
  						snake_case     :					int     =	[]
  						for k in range(len(UpperCamelCase__			)			):
  								if output[k][2] * output[k][3] >= thresh:
  										snake_case ,snake_case     :					Tuple     =	output[k][0], output[k][1]
  										snake_case     :					str     =	self.tokenizer.decode(W_query['''input_ids'''][idx][c_start_pos:c_end_pos]			)
  										temp.append((decoded, output[k][2] * output[k][3])			)
  
  						final_outputs.append(UpperCamelCase__			)
  
  				return final_outputs
  
  
  