code: 
  
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  	import torch
  
  	from transformers import (
  	    RemBertConfig,
  	    RemBertForCausalLM,
  	    RemBertForMaskedLM,
  	    RemBertForMultipleChoice,
  	    RemBertForQuestionAnswering,
  	    RemBertForSequenceClassification,
  	    RemBertForTokenClassification,
  	    RemBertModel,
  	)
  	from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class    __A				:
  
  
  
  
  
  
  				def __init__(	self			:    int    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Tuple=13    ,				UpperCAmelCase_			:    Tuple=7    ,				UpperCAmelCase_			:    int=True    ,				UpperCAmelCase_			:    Dict=True    ,				UpperCAmelCase_			:    List[Any]=True    ,				UpperCAmelCase_			:    str=True    ,				UpperCAmelCase_			:    Dict=99    ,				UpperCAmelCase_			:    str=32    ,				UpperCAmelCase_			:    Union[str, Any]=18    ,				UpperCAmelCase_			:    Any=43    ,				UpperCAmelCase_			:    Optional[int]=5    ,				UpperCAmelCase_			:    Tuple=4    ,				UpperCAmelCase_			:    List[str]=37    ,				UpperCAmelCase_			:    Tuple="gelu"    ,				UpperCAmelCase_			:    Optional[Any]=0.1    ,				UpperCAmelCase_			:    int=0.1    ,				UpperCAmelCase_			:    str=512    ,				UpperCAmelCase_			:    Optional[int]=16    ,				UpperCAmelCase_			:    List[Any]=2    ,				UpperCAmelCase_			:    int=0.02    ,				UpperCAmelCase_			:    str=3    ,				UpperCAmelCase_			:    str=4    ,				UpperCAmelCase_			:    Optional[int]=None    ,				):
  								lowerCAmelCase				:			Union[str, Any]    			=      parent
  								lowerCAmelCase				:			Any    			=      batch_size
  								lowerCAmelCase				:			Optional[Any]    			=      seq_length
  								lowerCAmelCase				:			str    			=      is_training
  								lowerCAmelCase				:			List[str]    			=      use_input_mask
  								lowerCAmelCase				:			Any    			=      use_token_type_ids
  								lowerCAmelCase				:			Union[str, Any]    			=      use_labels
  								lowerCAmelCase				:			Union[str, Any]    			=      vocab_size
  								lowerCAmelCase				:			Tuple    			=      hidden_size
  								lowerCAmelCase				:			Optional[Any]    			=      input_embedding_size
  								lowerCAmelCase				:			Optional[Any]    			=      output_embedding_size
  								lowerCAmelCase				:			Tuple    			=      num_hidden_layers
  								lowerCAmelCase				:			List[str]    			=      num_attention_heads
  								lowerCAmelCase				:			Union[str, Any]    			=      intermediate_size
  								lowerCAmelCase				:			Union[str, Any]    			=      hidden_act
  								lowerCAmelCase				:			Union[str, Any]    			=      hidden_dropout_prob
  								lowerCAmelCase				:			Dict    			=      attention_probs_dropout_prob
  								lowerCAmelCase				:			Optional[int]    			=      max_position_embeddings
  								lowerCAmelCase				:			Dict    			=      type_vocab_size
  								lowerCAmelCase				:			List[Any]    			=      type_sequence_label_size
  								lowerCAmelCase				:			Optional[int]    			=      initializer_range
  								lowerCAmelCase				:			Optional[Any]    			=      num_labels
  								lowerCAmelCase				:			int    			=      num_choices
  								lowerCAmelCase				:			Dict    			=      scope
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    List[Any]    ):
  								lowerCAmelCase				:			int    			=      ids_tensor([self.batch_size, self.seq_length]    ,				self.vocab_size    )
  
  								lowerCAmelCase				:			List[str]    			=      None
  								if self.use_input_mask:
  												lowerCAmelCase				:			List[Any]    			=      random_attention_mask([self.batch_size, self.seq_length]    )
  
  								lowerCAmelCase				:			List[Any]    			=      None
  								if self.use_token_type_ids:
  												lowerCAmelCase				:			int    			=      ids_tensor([self.batch_size, self.seq_length]    ,				self.type_vocab_size    )
  
  								lowerCAmelCase				:			List[Any]    			=      None
  								lowerCAmelCase				:			List[str]    			=      None
  								lowerCAmelCase				:			Dict    			=      None
  								if self.use_labels:
  												lowerCAmelCase				:			List[str]    			=      ids_tensor([self.batch_size]    ,				self.type_sequence_label_size    )
  												lowerCAmelCase				:			Union[str, Any]    			=      ids_tensor([self.batch_size, self.seq_length]    ,				self.num_labels    )
  												lowerCAmelCase				:			Dict    			=      ids_tensor([self.batch_size]    ,				self.num_choices    )
  
  								lowerCAmelCase				:			Optional[Any]    			=      RemBertConfig(
  								    vocab_size=self.vocab_size    ,				hidden_size=self.hidden_size    ,				input_embedding_size=self.input_embedding_size    ,				output_embedding_size=self.output_embedding_size    ,				num_hidden_layers=self.num_hidden_layers    ,				num_attention_heads=self.num_attention_heads    ,				intermediate_size=self.intermediate_size    ,				hidden_act=self.hidden_act    ,				hidden_dropout_prob=self.hidden_dropout_prob    ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,				max_position_embeddings=self.max_position_embeddings    ,				type_vocab_size=self.type_vocab_size    ,				is_decoder=UpperCAmelCase_    ,				initializer_range=self.initializer_range    ,				)
  
  								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    str    ):
  								(
  								    lowerCAmelCase  
  								)				:			Optional[Any]    			=      self.prepare_config_and_inputs()
  
  								lowerCAmelCase				:			Tuple    			=      True
  								lowerCAmelCase				:			Optional[Any]    			=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
  								lowerCAmelCase				:			str    			=      ids_tensor([self.batch_size, self.seq_length]    ,				vocab_size=2    )
  
  								return (
  								    config,
  								    input_ids,
  								    token_type_ids,
  								    input_mask,
  								    sequence_labels,
  								    token_labels,
  								    choice_labels,
  								    encoder_hidden_states,
  								    encoder_attention_mask,
  								)
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Tuple    ,				UpperCAmelCase_			:    Tuple    ,				UpperCAmelCase_			:    str    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    int    ):
  								lowerCAmelCase				:			str    			=      RemBertModel(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			Optional[int]    			=      model(UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    )
  								lowerCAmelCase				:			str    			=      model(UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    )
  								lowerCAmelCase				:			Optional[Any]    			=      model(UpperCAmelCase_    )
  								self.parent.assertEqual(result.last_hidden_state.shape    ,				(self.batch_size, self.seq_length, self.hidden_size)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    str    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Tuple    ,				UpperCAmelCase_			:    int    ,				UpperCAmelCase_			:    Union[str, Any]    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Union[str, Any]    ,				):
  								lowerCAmelCase				:			List[Any]    			=      True
  								lowerCAmelCase				:			Optional[int]    			=      RemBertModel(UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			int    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				encoder_hidden_states=UpperCAmelCase_    ,				encoder_attention_mask=UpperCAmelCase_    ,				)
  								lowerCAmelCase				:			Optional[int]    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				encoder_hidden_states=UpperCAmelCase_    ,				)
  								lowerCAmelCase				:			Optional[Any]    			=      model(UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    )
  								self.parent.assertEqual(result.last_hidden_state.shape    ,				(self.batch_size, self.seq_length, self.hidden_size)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    int    ,				UpperCAmelCase_			:    str    ,				UpperCAmelCase_			:    Union[str, Any]    ,				UpperCAmelCase_			:    Optional[int]    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Any    ,				):
  								lowerCAmelCase				:			Any    			=      RemBertForCausalLM(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			List[str]    			=      model(UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				labels=UpperCAmelCase_    )
  								self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.seq_length, self.vocab_size)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Union[str, Any]    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    Optional[Any]    ,				UpperCAmelCase_			:    str    ,				UpperCAmelCase_			:    Union[str, Any]    ):
  								lowerCAmelCase				:			Any    			=      RemBertForMaskedLM(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			List[str]    			=      model(UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				labels=UpperCAmelCase_    )
  								self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.seq_length, self.vocab_size)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ,				UpperCAmelCase_			:    Optional[int]    ,				UpperCAmelCase_			:    Optional[int]    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Tuple    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Union[str, Any]    ,				UpperCAmelCase_			:    Tuple    ,				):
  								lowerCAmelCase				:			Optional[Any]    			=      True
  								lowerCAmelCase				:			Dict    			=      True
  								lowerCAmelCase				:			List[Any]    			=      RemBertForCausalLM(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  
  								# first forward pass
  								lowerCAmelCase				:			int    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				encoder_hidden_states=UpperCAmelCase_    ,				encoder_attention_mask=UpperCAmelCase_    ,				use_cache=UpperCAmelCase_    ,				)
  								lowerCAmelCase				:			Optional[Any]    			=      outputs.past_key_values
  
  								# create hypothetical multiple next token and extent to next_input_ids
  								lowerCAmelCase				:			List[Any]    			=      ids_tensor((self.batch_size, 3)    ,				config.vocab_size    )
  								lowerCAmelCase				:			Tuple    			=      ids_tensor((self.batch_size, 3)    ,				vocab_size=2    )
  
  								# append to next input_ids and
  								lowerCAmelCase				:			int    			=      torch.cat([input_ids, next_tokens]    ,				dim=-1    )
  								lowerCAmelCase				:			Any    			=      torch.cat([input_mask, next_mask]    ,				dim=-1    )
  
  								lowerCAmelCase				:			List[Any]    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				encoder_hidden_states=UpperCAmelCase_    ,				encoder_attention_mask=UpperCAmelCase_    ,				output_hidden_states=UpperCAmelCase_    ,				)['hidden_states'][0]
  								lowerCAmelCase				:			Any    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				encoder_hidden_states=UpperCAmelCase_    ,				encoder_attention_mask=UpperCAmelCase_    ,				past_key_values=UpperCAmelCase_    ,				output_hidden_states=UpperCAmelCase_    ,				)['hidden_states'][0]
  
  								# select random slice
  								lowerCAmelCase				:			List[Any]    			=      ids_tensor((1,)    ,				output_from_past.shape[-1]    ).item()
  								lowerCAmelCase				:			Tuple    			=      output_from_no_past[:, -3:, random_slice_idx].detach()
  								lowerCAmelCase				:			Optional[Any]    			=      output_from_past[:, :, random_slice_idx].detach()
  
  								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )
  
  								# test that outputs are equal for slice
  								self.parent.assertTrue(torch.allclose(UpperCAmelCase_    ,				UpperCAmelCase_    ,				atol=1E-3    )    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Optional[int]    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Tuple    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    Optional[Any]    ):
  								lowerCAmelCase				:			List[str]    			=      RemBertForQuestionAnswering(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			Tuple    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				start_positions=UpperCAmelCase_    ,				end_positions=UpperCAmelCase_    ,				)
  								self.parent.assertEqual(result.start_logits.shape    ,				(self.batch_size, self.seq_length)    )
  								self.parent.assertEqual(result.end_logits.shape    ,				(self.batch_size, self.seq_length)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    int    ,				UpperCAmelCase_			:    Dict    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    Union[str, Any]    ,				UpperCAmelCase_			:    Union[str, Any]    ,				UpperCAmelCase_			:    Optional[Any]    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    Any    ):
  								lowerCAmelCase				:			Any    			=      self.num_labels
  								lowerCAmelCase				:			Tuple    			=      RemBertForSequenceClassification(UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			Optional[int]    			=      model(UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				labels=UpperCAmelCase_    )
  								self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.num_labels)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    List[str]    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    Optional[Any]    ,				UpperCAmelCase_			:    int    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    Tuple    ,				UpperCAmelCase_			:    List[Any]    ):
  								lowerCAmelCase				:			List[str]    			=      self.num_labels
  								lowerCAmelCase				:			List[Any]    			=      RemBertForTokenClassification(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			Union[str, Any]    			=      model(UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				labels=UpperCAmelCase_    )
  								self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.seq_length, self.num_labels)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    List[str]    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    Union[str, Any]    ,				UpperCAmelCase_			:    List[str]    ,				UpperCAmelCase_			:    int    ,				UpperCAmelCase_			:    List[Any]    ,				UpperCAmelCase_			:    Union[str, Any]    ):
  								lowerCAmelCase				:			Optional[Any]    			=      self.num_choices
  								lowerCAmelCase				:			int    			=      RemBertForMultipleChoice(config=UpperCAmelCase_    )
  								model.to(UpperCAmelCase_    )
  								model.eval()
  								lowerCAmelCase				:			Tuple    			=      input_ids.unsqueeze(1    ).expand(-1    ,				self.num_choices    ,				-1    ).contiguous()
  								lowerCAmelCase				:			Tuple    			=      token_type_ids.unsqueeze(1    ).expand(-1    ,				self.num_choices    ,				-1    ).contiguous()
  								lowerCAmelCase				:			List[Any]    			=      input_mask.unsqueeze(1    ).expand(-1    ,				self.num_choices    ,				-1    ).contiguous()
  								lowerCAmelCase				:			List[str]    			=      model(
  								    UpperCAmelCase_    ,				attention_mask=UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				labels=UpperCAmelCase_    ,				)
  								self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.num_choices)    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ):
  								lowerCAmelCase				:			Optional[int]    			=      self.prepare_config_and_inputs()
  								(
  								    lowerCAmelCase  
  								)				:			List[Any]    			=      config_and_inputs
  								lowerCAmelCase				:			str    			=      {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
  								return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class    __A				(							lowerCAmelCase						,						lowerCAmelCase						,						unittest.TestCase ):
  				lowerCAmelCase_							:		str						=							(
  				    (
  				        RemBertModel,
  				        RemBertForMaskedLM,
  				        RemBertForCausalLM,
  				        RemBertForMultipleChoice,
  				        RemBertForQuestionAnswering,
  				        RemBertForSequenceClassification,
  				        RemBertForTokenClassification,
  				    )
  				    if is_torch_available()
  				    else ()
  				)
  				lowerCAmelCase_							:		Any						=							(RemBertForCausalLM,) if is_torch_available() else ()
  				lowerCAmelCase_							:		List[Any]						=							(
  				    {
  				        "feature-extraction": RemBertModel,
  				        "fill-mask": RemBertForMaskedLM,
  				        "question-answering": RemBertForQuestionAnswering,
  				        "text-classification": RemBertForSequenceClassification,
  				        "text-generation": RemBertForCausalLM,
  				        "token-classification": RemBertForTokenClassification,
  				        "zero-shot": RemBertForSequenceClassification,
  				    }
  				    if is_torch_available()
  				    else {}
  				)
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Optional[int]    ):
  								lowerCAmelCase				:			str    			=      RemBertModelTester(self    )
  								lowerCAmelCase				:			List[str]    			=      ConfigTester(self    ,				config_class=UpperCAmelCase_    ,				hidden_size=37    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ):
  								self.config_tester.run_common_tests()
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Any    ):
  								lowerCAmelCase				:			Union[str, Any]    			=      self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_model(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    List[Any]    ):
  								lowerCAmelCase				:			List[Any]    			=      self.model_tester.prepare_config_and_inputs()
  								for type in ["absolute", "relative_key", "relative_key_query"]:
  												lowerCAmelCase				:			str    			=      type
  												self.model_tester.create_and_check_model(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    str    ):
  								lowerCAmelCase				:			Tuple    			=      self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ):
  								lowerCAmelCase				:			Tuple    			=      self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Tuple    ):
  								lowerCAmelCase				:			Dict    			=      self.model_tester.prepare_config_and_inputs_for_decoder()
  								self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    List[str]    ):
  								lowerCAmelCase				:			Dict    			=      self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Any    ):
  								lowerCAmelCase				:			List[Any]    			=      self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Dict    ):
  								lowerCAmelCase				:			Any    			=      self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Optional[Any]    ):
  								lowerCAmelCase				:			Optional[int]    			=      self.model_tester.prepare_config_and_inputs_for_decoder()
  								self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase_    )
  
  
  
  
  
  
  				def 	lowercase__ (	self			:    Optional[int]    ):
  								# This regression test was failing with PyTorch < 1.3
  								(
  								    lowerCAmelCase  
  								)				:			Union[str, Any]    			=      self.model_tester.prepare_config_and_inputs_for_decoder()
  
  								lowerCAmelCase				:			List[str]    			=      None
  
  								self.model_tester.create_and_check_model_as_decoder(
  								    UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				)
  
  
  
  
  
  
  				@slow
  				def 	lowercase__ (	self			:    Tuple    ):
  								for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  												lowerCAmelCase				:			int    			=      RemBertModel.from_pretrained(UpperCAmelCase_    )
  												self.assertIsNotNone(UpperCAmelCase_    )
  
  
  
  
  
  
  @require_torch
  class    __A				(							unittest.TestCase ):
  
  
  
  
  
  
  				@slow
  				def 	lowercase__ (	self			:    List[Any]    ):
  								# Test exact values at the last hidden layer
  								lowerCAmelCase				:			Union[str, Any]    			=      RemBertModel.from_pretrained('google/rembert'    )
  								lowerCAmelCase				:			str    			=      torch.tensor([[312, 56498, 313, 2125, 313]]    )
  								lowerCAmelCase				:			List[str]    			=      torch.tensor([[0, 0, 0, 1, 1]]    )
  								with torch.no_grad():
  												lowerCAmelCase				:			str    			=      model(UpperCAmelCase_    ,				token_type_ids=UpperCAmelCase_    ,				output_hidden_states=UpperCAmelCase_    )
  
  								lowerCAmelCase				:			int    			=      1152
  
  								lowerCAmelCase				:			Union[str, Any]    			=      torch.Size((1, 5, hidden_size)    )
  								self.assertEqual(output['last_hidden_state'].shape    ,				UpperCAmelCase_    )
  
  								lowerCAmelCase				:			Dict    			=      torch.tensor(
  								    [
  								        [
  								            [0.07_54, -0.20_22, 0.19_04],
  								            [-0.33_54, -0.36_92, -0.47_91],
  								            [-0.23_14, -0.67_29, -0.07_49],
  								            [-0.03_96, -0.31_05, -0.42_34],
  								            [-0.15_71, -0.05_25, 0.53_53],
  								        ]
  								    ]    )
  
  								# Running on the original tf implementation gives slightly different results here.
  								# Not clear why this variations is present
  								# TODO: Find reason for discrepancy
  								# expected_original_implementation = [[
  								#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
  								#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
  								#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
  								#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
  								#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
  								# ]]
  
  								self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]    ,				UpperCAmelCase_    ,				atol=1E-4    )    )
  
  
  
  
  
  
style_context: 
  
  import torch
  
  from diffusers import KDPMaAncestralDiscreteScheduler
  from diffusers.utils import torch_device
  
  from .test_schedulers import SchedulerCommonTest
  
  
  
  
  
  
  class    __A				(							lowerCAmelCase ):
      lowerCAmelCase_							:		List[str]						=							(KDPMaAncestralDiscreteScheduler,)
      lowerCAmelCase_							:		Dict						=							10
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Any    ,				**UpperCAmelCase_			:    List[Any]    ):
          lowerCAmelCase				:			int    			=      {
              'num_train_timesteps': 1100,
              'beta_start': 0.00_01,
              'beta_end': 0.02,
              'beta_schedule': 'linear',
          }
  
          config.update(**UpperCAmelCase_    )
          return config
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Optional[Any]    ):
          for timesteps in [10, 50, 100, 1000]:
              self.check_over_configs(num_train_timesteps=UpperCAmelCase_    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          for beta_start, beta_end in zip([0.0_00_01, 0.00_01, 0.0_01]    ,				[0.00_02, 0.0_02, 0.02]    ):
              self.check_over_configs(beta_start=UpperCAmelCase_    ,				beta_end=UpperCAmelCase_    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Optional[int]    ):
          for schedule in ["linear", "scaled_linear"]:
              self.check_over_configs(beta_schedule=UpperCAmelCase_    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    int    ):
          if torch_device == "mps":
              return
          lowerCAmelCase				:			Any    			=      self.scheduler_classes[0]
          lowerCAmelCase				:			Optional[int]    			=      self.get_scheduler_config()
          lowerCAmelCase				:			List[str]    			=      scheduler_class(**UpperCAmelCase_    )
  
          scheduler.set_timesteps(self.num_inference_steps    )
  
          lowerCAmelCase				:			Any    			=      torch.manual_seed(0    )
  
          lowerCAmelCase				:			List[Any]    			=      self.dummy_model()
          lowerCAmelCase				:			List[Any]    			=      self.dummy_sample_deter * scheduler.init_noise_sigma
          lowerCAmelCase				:			Optional[int]    			=      sample.to(UpperCAmelCase_    )
  
          for i, t in enumerate(scheduler.timesteps    ):
              lowerCAmelCase				:			str    			=      scheduler.scale_model_input(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
              lowerCAmelCase				:			Any    			=      model(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
              lowerCAmelCase				:			Optional[Any]    			=      scheduler.step(UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				generator=UpperCAmelCase_    )
              lowerCAmelCase				:			Union[str, Any]    			=      output.prev_sample
  
          lowerCAmelCase				:			List[Any]    			=      torch.sum(torch.abs(UpperCAmelCase_    )    )
          lowerCAmelCase				:			Any    			=      torch.mean(torch.abs(UpperCAmelCase_    )    )
  
          assert abs(result_sum.item() - 1_38_49.38_77    ) < 1E-2
          assert abs(result_mean.item() - 18.03_31    ) < 5E-3
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Tuple    ):
          for prediction_type in ["epsilon", "v_prediction"]:
              self.check_over_configs(prediction_type=UpperCAmelCase_    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          if torch_device == "mps":
              return
          lowerCAmelCase				:			Optional[Any]    			=      self.scheduler_classes[0]
          lowerCAmelCase				:			List[Any]    			=      self.get_scheduler_config(prediction_type='v_prediction'    )
          lowerCAmelCase				:			Any    			=      scheduler_class(**UpperCAmelCase_    )
  
          scheduler.set_timesteps(self.num_inference_steps    )
  
          lowerCAmelCase				:			Optional[int]    			=      self.dummy_model()
          lowerCAmelCase				:			Any    			=      self.dummy_sample_deter * scheduler.init_noise_sigma
          lowerCAmelCase				:			Optional[int]    			=      sample.to(UpperCAmelCase_    )
  
          lowerCAmelCase				:			Optional[int]    			=      torch.manual_seed(0    )
  
          for i, t in enumerate(scheduler.timesteps    ):
              lowerCAmelCase				:			Dict    			=      scheduler.scale_model_input(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
              lowerCAmelCase				:			Optional[int]    			=      model(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
              lowerCAmelCase				:			Optional[int]    			=      scheduler.step(UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				generator=UpperCAmelCase_    )
              lowerCAmelCase				:			str    			=      output.prev_sample
  
          lowerCAmelCase				:			str    			=      torch.sum(torch.abs(UpperCAmelCase_    )    )
          lowerCAmelCase				:			Tuple    			=      torch.mean(torch.abs(UpperCAmelCase_    )    )
  
          assert abs(result_sum.item() - 3_28.99_70    ) < 1E-2
          assert abs(result_mean.item() - 0.42_84    ) < 1E-3
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Tuple    ):
          if torch_device == "mps":
              return
          lowerCAmelCase				:			str    			=      self.scheduler_classes[0]
          lowerCAmelCase				:			Dict    			=      self.get_scheduler_config()
          lowerCAmelCase				:			List[str]    			=      scheduler_class(**UpperCAmelCase_    )
  
          scheduler.set_timesteps(self.num_inference_steps    ,				device=UpperCAmelCase_    )
          lowerCAmelCase				:			int    			=      torch.manual_seed(0    )
  
          lowerCAmelCase				:			Tuple    			=      self.dummy_model()
          lowerCAmelCase				:			Optional[Any]    			=      self.dummy_sample_deter.to(UpperCAmelCase_    ) * scheduler.init_noise_sigma
  
          for t in scheduler.timesteps:
              lowerCAmelCase				:			str    			=      scheduler.scale_model_input(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
              lowerCAmelCase				:			List[Any]    			=      model(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
              lowerCAmelCase				:			Dict    			=      scheduler.step(UpperCAmelCase_    ,				UpperCAmelCase_    ,				UpperCAmelCase_    ,				generator=UpperCAmelCase_    )
              lowerCAmelCase				:			Dict    			=      output.prev_sample
  
          lowerCAmelCase				:			Union[str, Any]    			=      torch.sum(torch.abs(UpperCAmelCase_    )    )
          lowerCAmelCase				:			str    			=      torch.mean(torch.abs(UpperCAmelCase_    )    )
  
          assert abs(result_sum.item() - 1_38_49.38_18    ) < 1E-1
          assert abs(result_mean.item() - 18.03_31    ) < 1E-3
  
  
  
  
  
  