code: import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  	import torch
  
  	from transformers import (
  	    RemBertConfig,
  	    RemBertForCausalLM,
  	    RemBertForMaskedLM,
  	    RemBertForMultipleChoice,
  	    RemBertForQuestionAnswering,
  	    RemBertForSequenceClassification,
  	    RemBertForTokenClassification,
  	    RemBertModel,
  	)
  	from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  class      lowercase_						:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=13	,    __SCREAMING_SNAKE_CASE=7	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=99	,    __SCREAMING_SNAKE_CASE=32	,    __SCREAMING_SNAKE_CASE=18	,    __SCREAMING_SNAKE_CASE=43	,    __SCREAMING_SNAKE_CASE=5	,    __SCREAMING_SNAKE_CASE=4	,    __SCREAMING_SNAKE_CASE=37	,    __SCREAMING_SNAKE_CASE="gelu"	,    __SCREAMING_SNAKE_CASE=0.1	,    __SCREAMING_SNAKE_CASE=0.1	,    __SCREAMING_SNAKE_CASE=512	,    __SCREAMING_SNAKE_CASE=16	,    __SCREAMING_SNAKE_CASE=2	,    __SCREAMING_SNAKE_CASE=0.0_2	,    __SCREAMING_SNAKE_CASE=3	,    __SCREAMING_SNAKE_CASE=4	,    __SCREAMING_SNAKE_CASE=None	,    )		->List[Any]:
  								lowerCAmelCase				=	parent
  								lowerCAmelCase				=	batch_size
  								lowerCAmelCase				=	seq_length
  								lowerCAmelCase				=	is_training
  								lowerCAmelCase				=	use_input_mask
  								lowerCAmelCase				=	use_token_type_ids
  								lowerCAmelCase				=	use_labels
  								lowerCAmelCase				=	vocab_size
  								lowerCAmelCase				=	hidden_size
  								lowerCAmelCase				=	input_embedding_size
  								lowerCAmelCase				=	output_embedding_size
  								lowerCAmelCase				=	num_hidden_layers
  								lowerCAmelCase				=	num_attention_heads
  								lowerCAmelCase				=	intermediate_size
  								lowerCAmelCase				=	hidden_act
  								lowerCAmelCase				=	hidden_dropout_prob
  								lowerCAmelCase				=	attention_probs_dropout_prob
  								lowerCAmelCase				=	max_position_embeddings
  								lowerCAmelCase				=	type_vocab_size
  								lowerCAmelCase				=	type_sequence_label_size
  								lowerCAmelCase				=	initializer_range
  								lowerCAmelCase				=	num_labels
  								lowerCAmelCase				=	num_choices
  								lowerCAmelCase				=	scope
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								lowerCAmelCase				=	ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size       )
  
  								lowerCAmelCase				=	None
  								if self.use_input_mask:
  															lowerCAmelCase				=	random_attention_mask([self.batch_size, self.seq_length]       )
  
  								lowerCAmelCase				=	None
  								if self.use_token_type_ids:
  															lowerCAmelCase				=	ids_tensor([self.batch_size, self.seq_length]	,    self.type_vocab_size       )
  
  								lowerCAmelCase				=	None
  								lowerCAmelCase				=	None
  								lowerCAmelCase				=	None
  								if self.use_labels:
  															lowerCAmelCase				=	ids_tensor([self.batch_size]	,    self.type_sequence_label_size       )
  															lowerCAmelCase				=	ids_tensor([self.batch_size, self.seq_length]	,    self.num_labels       )
  															lowerCAmelCase				=	ids_tensor([self.batch_size]	,    self.num_choices       )
  
  								lowerCAmelCase				=	RemBertConfig(
  								    vocab_size=self.vocab_size	,    hidden_size=self.hidden_size	,    input_embedding_size=self.input_embedding_size	,    output_embedding_size=self.output_embedding_size	,    num_hidden_layers=self.num_hidden_layers	,    num_attention_heads=self.num_attention_heads	,    intermediate_size=self.intermediate_size	,    hidden_act=self.hidden_act	,    hidden_dropout_prob=self.hidden_dropout_prob	,    attention_probs_dropout_prob=self.attention_probs_dropout_prob	,    max_position_embeddings=self.max_position_embeddings	,    type_vocab_size=self.type_vocab_size	,    is_decoder=__SCREAMING_SNAKE_CASE	,    initializer_range=self.initializer_range	,    )
  
  								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								(
  								    (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   
  								)				=	self.prepare_config_and_inputs()
  
  								lowerCAmelCase				=	True
  								lowerCAmelCase				=	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
  								lowerCAmelCase				=	ids_tensor([self.batch_size, self.seq_length]	,    vocab_size=2       )
  
  								return (
  								    config,
  								    input_ids,
  								    token_type_ids,
  								    input_mask,
  								    sequence_labels,
  								    token_labels,
  								    choice_labels,
  								    encoder_hidden_states,
  								    encoder_attention_mask,
  								)
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->str:
  								lowerCAmelCase				=	RemBertModel(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE       )
  								self.parent.assertEqual(result.last_hidden_state.shape	,    (self.batch_size, self.seq_length, self.hidden_size)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    )		->str:
  								lowerCAmelCase				=	True
  								lowerCAmelCase				=	RemBertModel(__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    encoder_hidden_states=__SCREAMING_SNAKE_CASE	,    encoder_attention_mask=__SCREAMING_SNAKE_CASE	,    )
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    encoder_hidden_states=__SCREAMING_SNAKE_CASE	,    )
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE       )
  								self.parent.assertEqual(result.last_hidden_state.shape	,    (self.batch_size, self.seq_length, self.hidden_size)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    )		->int:
  								lowerCAmelCase				=	RemBertForCausalLM(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE       )
  								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.vocab_size)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Optional[Any]:
  								lowerCAmelCase				=	RemBertForMaskedLM(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE       )
  								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.vocab_size)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    )		->Optional[int]:
  								lowerCAmelCase				=	True
  								lowerCAmelCase				=	True
  								lowerCAmelCase				=	RemBertForCausalLM(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  
  								# first forward pass
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    encoder_hidden_states=__SCREAMING_SNAKE_CASE	,    encoder_attention_mask=__SCREAMING_SNAKE_CASE	,    use_cache=__SCREAMING_SNAKE_CASE	,    )
  								lowerCAmelCase				=	outputs.past_key_values
  
  								# create hypothetical multiple next token and extent to next_input_ids
  								lowerCAmelCase				=	ids_tensor((self.batch_size, 3)	,    config.vocab_size       )
  								lowerCAmelCase				=	ids_tensor((self.batch_size, 3)	,    vocab_size=2       )
  
  								# append to next input_ids and
  								lowerCAmelCase				=	torch.cat([input_ids, next_tokens]	,    dim=-1       )
  								lowerCAmelCase				=	torch.cat([input_mask, next_mask]	,    dim=-1       )
  
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    encoder_hidden_states=__SCREAMING_SNAKE_CASE	,    encoder_attention_mask=__SCREAMING_SNAKE_CASE	,    output_hidden_states=__SCREAMING_SNAKE_CASE	,    )['''hidden_states'''][0]
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    encoder_hidden_states=__SCREAMING_SNAKE_CASE	,    encoder_attention_mask=__SCREAMING_SNAKE_CASE	,    past_key_values=__SCREAMING_SNAKE_CASE	,    output_hidden_states=__SCREAMING_SNAKE_CASE	,    )['''hidden_states'''][0]
  
  								# select random slice
  								lowerCAmelCase				=	ids_tensor((1,)	,    output_from_past.shape[-1]       ).item()
  								lowerCAmelCase				=	output_from_no_past[:, -3:, random_slice_idx].detach()
  								lowerCAmelCase				=	output_from_past[:, :, random_slice_idx].detach()
  
  								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )
  
  								# test that outputs are equal for slice
  								self.parent.assertTrue(torch.allclose(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    atol=1e-3       )       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Optional[Any]:
  								lowerCAmelCase				=	RemBertForQuestionAnswering(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    start_positions=__SCREAMING_SNAKE_CASE	,    end_positions=__SCREAMING_SNAKE_CASE	,    )
  								self.parent.assertEqual(result.start_logits.shape	,    (self.batch_size, self.seq_length)       )
  								self.parent.assertEqual(result.end_logits.shape	,    (self.batch_size, self.seq_length)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->List[str]:
  								lowerCAmelCase				=	self.num_labels
  								lowerCAmelCase				=	RemBertForSequenceClassification(__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE       )
  								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_labels)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->List[Any]:
  								lowerCAmelCase				=	self.num_labels
  								lowerCAmelCase				=	RemBertForTokenClassification(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE       )
  								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.num_labels)       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Optional[Any]:
  								lowerCAmelCase				=	self.num_choices
  								lowerCAmelCase				=	RemBertForMultipleChoice(config=__SCREAMING_SNAKE_CASE       )
  								model.to(__SCREAMING_SNAKE_CASE       )
  								model.eval()
  								lowerCAmelCase				=	input_ids.unsqueeze(1       ).expand(-1	,    self.num_choices	,    -1       ).contiguous()
  								lowerCAmelCase				=	token_type_ids.unsqueeze(1       ).expand(-1	,    self.num_choices	,    -1       ).contiguous()
  								lowerCAmelCase				=	input_mask.unsqueeze(1       ).expand(-1	,    self.num_choices	,    -1       ).contiguous()
  								lowerCAmelCase				=	model(
  								    __SCREAMING_SNAKE_CASE	,    attention_mask=__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE	,    )
  								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_choices)       )
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
  								lowerCAmelCase				=	self.prepare_config_and_inputs()
  								(
  								    (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   
  								)				=	config_and_inputs
  								lowerCAmelCase				=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
  								return config, inputs_dict
  
  
  
  
  @require_torch
  class      lowercase_						(						UpperCamelCase_   ,  UpperCamelCase_   ,  unittest.TestCase		):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	UpperCAmelCase_						:      int     =       (
  	    (
  	        RemBertModel,
  	        RemBertForMaskedLM,
  	        RemBertForCausalLM,
  	        RemBertForMultipleChoice,
  	        RemBertForQuestionAnswering,
  	        RemBertForSequenceClassification,
  	        RemBertForTokenClassification,
  	    )
  	    if is_torch_available()
  	    else ()
  	)
  	UpperCAmelCase_						:      Dict     =       (RemBertForCausalLM,) if is_torch_available() else ()
  	UpperCAmelCase_						:      List[Any]     =       (
  	    {
  	        """feature-extraction""": RemBertModel,
  	        """fill-mask""": RemBertForMaskedLM,
  	        """question-answering""": RemBertForQuestionAnswering,
  	        """text-classification""": RemBertForSequenceClassification,
  	        """text-generation""": RemBertForCausalLM,
  	        """token-classification""": RemBertForTokenClassification,
  	        """zero-shot""": RemBertForSequenceClassification,
  	    }
  	    if is_torch_available()
  	    else {}
  	)
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								lowerCAmelCase				=	RemBertModelTester(self       )
  								lowerCAmelCase				=	ConfigTester(self	,    config_class=__SCREAMING_SNAKE_CASE	,    hidden_size=37       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->str:
  								self.config_tester.run_common_tests()
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[Any]:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Union[str, Any]:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								for type in ["absolute", "relative_key", "relative_key_query"]:
  															lowerCAmelCase				=	type
  															self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Tuple:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs_for_decoder()
  								self.model_tester.create_and_check_decoder_model_past_large_inputs(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Union[str, Any]:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs_for_decoder()
  								self.model_tester.create_and_check_model_as_decoder(*__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[str]:
  								# This regression test was failing with PyTorch < 1.3
  								(
  								    (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   (
  								    lowerCAmelCase
  								)	,   
  								)				=	self.model_tester.prepare_config_and_inputs_for_decoder()
  
  								lowerCAmelCase				=	None
  
  								self.model_tester.create_and_check_model_as_decoder(
  								    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    )
  
  
  
  
  	@slow
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  															lowerCAmelCase				=	RemBertModel.from_pretrained(__SCREAMING_SNAKE_CASE       )
  															self.assertIsNotNone(__SCREAMING_SNAKE_CASE       )
  
  
  
  
  @require_torch
  class      lowercase_						(						unittest.TestCase		):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	@slow
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
  								# Test exact values at the last hidden layer
  								lowerCAmelCase				=	RemBertModel.from_pretrained('''google/rembert'''       )
  								lowerCAmelCase				=	torch.tensor([[312, 56498, 313, 2125, 313]]       )
  								lowerCAmelCase				=	torch.tensor([[0, 0, 0, 1, 1]]       )
  								with torch.no_grad():
  															lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    token_type_ids=__SCREAMING_SNAKE_CASE	,    output_hidden_states=__SCREAMING_SNAKE_CASE       )
  
  								lowerCAmelCase				=	1152
  
  								lowerCAmelCase				=	torch.Size((1, 5, hidden_size)       )
  								self.assertEqual(output['''last_hidden_state'''].shape	,    __SCREAMING_SNAKE_CASE       )
  
  								lowerCAmelCase				=	torch.tensor(
  								    [
  								        [
  								            [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
  								            [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
  								            [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
  								            [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
  								            [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
  								        ]
  								    ]       )
  
  								# Running on the original tf implementation gives slightly different results here.
  								# Not clear why this variations is present
  								# TODO: Find reason for discrepancy
  								# expected_original_implementation = [[
  								#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
  								#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
  								#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
  								#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
  								#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
  								# ]]
  
  								self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]	,    __SCREAMING_SNAKE_CASE	,    atol=1e-4       )       )
  
  
  
  
style_context: import torch
  
  from accelerate import Accelerator, DistributedType
  
  
  
  
  class      lowercase_						:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	def __enter__(							self       )		->Dict:
  								if self.enabled:
  															lowerCAmelCase				=	self.model.no_sync()
  															self.model_sync_obj.__enter__()
  
  								return self
  
  
  	def __exit__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Any:
  								if self.enabled:
  															# Average all models on exit
  															self._sync_and_avg_model_params()
  															self.model_sync_obj.__exit__(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = True       )		->Tuple:
  								if accelerator.distributed_type not in [
  								    DistributedType.NO,
  								    DistributedType.MULTI_CPU,
  								    DistributedType.MULTI_GPU,
  								]:
  															raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''       )
  								lowerCAmelCase				=	enabled and accelerator.distributed_type != DistributedType.NO
  								lowerCAmelCase				=	0
  								if self.enabled:
  															lowerCAmelCase				=	accelerator
  															lowerCAmelCase				=	model
  															lowerCAmelCase				=	local_sgd_steps
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[Any]:
  								self.num_steps += 1
  								if not self.enabled:
  															return
  
  								if self.num_steps % self.local_sgd_steps == 0:
  															self._sync_and_avg_model_params()
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
  								self.accelerator.wait_for_everyone()
  								with self.accelerator.autocast():
  															for param in self.model.parameters():
  																						lowerCAmelCase				=	self.accelerator.reduce(param.data	,    reduction='''mean'''       )
  
  
  
  