code: 
  
  
  
  
  
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  		import torch
  
  		from transformers import (
  		    RemBertConfig,
  		    RemBertForCausalLM,
  		    RemBertForMaskedLM,
  		    RemBertForMultipleChoice,
  		    RemBertForQuestionAnswering,
  		    RemBertForSequenceClassification,
  		    RemBertForTokenClassification,
  		    RemBertModel,
  		)
  		from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class 	UpperCamelCase_     :
  
  
  
  					def __init__(		self    :    int					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Dict=13					,			lowerCamelCase    :    Union[str, Any]=7					,			lowerCamelCase    :    Any=True					,			lowerCamelCase    :    int=True					,			lowerCamelCase    :    int=True					,			lowerCamelCase    :    int=True					,			lowerCamelCase    :    Union[str, Any]=99					,			lowerCamelCase    :    Any=32					,			lowerCamelCase    :    int=18					,			lowerCamelCase    :    Optional[Any]=43					,			lowerCamelCase    :    int=5					,			lowerCamelCase    :    str=4					,			lowerCamelCase    :    Tuple=37					,			lowerCamelCase    :    Any="gelu"					,			lowerCamelCase    :    Optional[int]=0.1					,			lowerCamelCase    :    Optional[Any]=0.1					,			lowerCamelCase    :    Optional[Any]=5_12					,			lowerCamelCase    :    Optional[int]=16					,			lowerCamelCase    :    Any=2					,			lowerCamelCase    :    Union[str, Any]=0.02					,			lowerCamelCase    :    Union[str, Any]=3					,			lowerCamelCase    :    Tuple=4					,			lowerCamelCase    :    int=None					,			):
  							lowerCamelCase_  :    int   	=			parent
  							lowerCamelCase_  :    int   	=			batch_size
  							lowerCamelCase_  :    List[str]   	=			seq_length
  							lowerCamelCase_  :    Optional[int]   	=			is_training
  							lowerCamelCase_  :    int   	=			use_input_mask
  							lowerCamelCase_  :    List[str]   	=			use_token_type_ids
  							lowerCamelCase_  :    Union[str, Any]   	=			use_labels
  							lowerCamelCase_  :    Tuple   	=			vocab_size
  							lowerCamelCase_  :    int   	=			hidden_size
  							lowerCamelCase_  :    Tuple   	=			input_embedding_size
  							lowerCamelCase_  :    Any   	=			output_embedding_size
  							lowerCamelCase_  :    Tuple   	=			num_hidden_layers
  							lowerCamelCase_  :    Dict   	=			num_attention_heads
  							lowerCamelCase_  :    Union[str, Any]   	=			intermediate_size
  							lowerCamelCase_  :    int   	=			hidden_act
  							lowerCamelCase_  :    str   	=			hidden_dropout_prob
  							lowerCamelCase_  :    str   	=			attention_probs_dropout_prob
  							lowerCamelCase_  :    Tuple   	=			max_position_embeddings
  							lowerCamelCase_  :    Optional[Any]   	=			type_vocab_size
  							lowerCamelCase_  :    List[str]   	=			type_sequence_label_size
  							lowerCamelCase_  :    int   	=			initializer_range
  							lowerCamelCase_  :    List[str]   	=			num_labels
  							lowerCamelCase_  :    List[Any]   	=			num_choices
  							lowerCamelCase_  :    Tuple   	=			scope
  
  
  
  					def      __a    (		self    :    str	):
  							lowerCamelCase_  :    Union[str, Any]   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.vocab_size	)
  
  							lowerCamelCase_  :    int   	=			None
  							if self.use_input_mask:
  									lowerCamelCase_  :    str   	=			random_attention_mask([self.batch_size, self.seq_length]	)
  
  							lowerCamelCase_  :    str   	=			None
  							if self.use_token_type_ids:
  									lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.type_vocab_size	)
  
  							lowerCamelCase_  :    Union[str, Any]   	=			None
  							lowerCamelCase_  :    Union[str, Any]   	=			None
  							lowerCamelCase_  :    List[str]   	=			None
  							if self.use_labels:
  									lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size]					,			self.type_sequence_label_size	)
  									lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.num_labels	)
  									lowerCamelCase_  :    List[str]   	=			ids_tensor([self.batch_size]					,			self.num_choices	)
  
  							lowerCamelCase_  :    Optional[int]   	=			RemBertConfig(
  							    vocab_size=self.vocab_size					,			hidden_size=self.hidden_size					,			input_embedding_size=self.input_embedding_size					,			output_embedding_size=self.output_embedding_size					,			num_hidden_layers=self.num_hidden_layers					,			num_attention_heads=self.num_attention_heads					,			intermediate_size=self.intermediate_size					,			hidden_act=self.hidden_act					,			hidden_dropout_prob=self.hidden_dropout_prob					,			attention_probs_dropout_prob=self.attention_probs_dropout_prob					,			max_position_embeddings=self.max_position_embeddings					,			type_vocab_size=self.type_vocab_size					,			is_decoder=lowerCamelCase					,			initializer_range=self.initializer_range					,			)
  
  							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  					def      __a    (		self    :    Optional[Any]	):
  							(
  							    lowerCamelCase_			
  							)  :    int   	=			self.prepare_config_and_inputs()
  
  							lowerCamelCase_  :    List[Any]   	=			True
  							lowerCamelCase_  :    Optional[int]   	=			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
  							lowerCamelCase_  :    Optional[Any]   	=			ids_tensor([self.batch_size, self.seq_length]					,			vocab_size=2	)
  
  							return (
  							    config,
  							    input_ids,
  							    token_type_ids,
  							    input_mask,
  							    sequence_labels,
  							    token_labels,
  							    choice_labels,
  							    encoder_hidden_states,
  							    encoder_attention_mask,
  							)
  
  
  
  					def      __a    (		self    :    str					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]	):
  							lowerCamelCase_  :    str   	=			RemBertModel(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    Any   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase	)
  							lowerCamelCase_  :    str   	=			model(lowerCamelCase					,			token_type_ids=lowerCamelCase	)
  							lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase	)
  							self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)	)
  
  
  
  					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    List[str]					,			):
  							lowerCamelCase_  :    Optional[int]   	=			True
  							lowerCamelCase_  :    Union[str, Any]   	=			RemBertModel(lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    Tuple   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			)
  							lowerCamelCase_  :    int   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			)
  							lowerCamelCase_  :    int   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase	)
  							self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)	)
  
  
  
  					def      __a    (		self    :    int					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[Any]					,			):
  							lowerCamelCase_  :    Any   	=			RemBertForCausalLM(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
  							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)	)
  
  
  
  					def      __a    (		self    :    Any					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    List[Any]	):
  							lowerCamelCase_  :    str   	=			RemBertForMaskedLM(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    int   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
  							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)	)
  
  
  
  					def      __a    (		self    :    int					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Dict					,			):
  							lowerCamelCase_  :    List[Any]   	=			True
  							lowerCamelCase_  :    Tuple   	=			True
  							lowerCamelCase_  :    List[str]   	=			RemBertForCausalLM(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  
  							# first forward pass
  							lowerCamelCase_  :    Optional[int]   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			use_cache=lowerCamelCase					,			)
  							lowerCamelCase_  :    List[str]   	=			outputs.past_key_values
  
  							# create hypothetical multiple next token and extent to next_input_ids
  							lowerCamelCase_  :    List[str]   	=			ids_tensor((self.batch_size, 3)					,			config.vocab_size	)
  							lowerCamelCase_  :    Dict   	=			ids_tensor((self.batch_size, 3)					,			vocab_size=2	)
  
  							# append to next input_ids and
  							lowerCamelCase_  :    Dict   	=			torch.cat([input_ids, next_tokens]					,			dim=-1	)
  							lowerCamelCase_  :    List[Any]   	=			torch.cat([input_mask, next_mask]					,			dim=-1	)
  
  							lowerCamelCase_  :    Optional[Any]   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			)['hidden_states'][0]
  							lowerCamelCase_  :    Union[str, Any]   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			past_key_values=lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			)['hidden_states'][0]
  
  							# select random slice
  							lowerCamelCase_  :    int   	=			ids_tensor((1,)					,			output_from_past.shape[-1]	).item()
  							lowerCamelCase_  :    str   	=			output_from_no_past[:, -3:, random_slice_idx].detach()
  							lowerCamelCase_  :    List[Any]   	=			output_from_past[:, :, random_slice_idx].detach()
  
  							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)
  
  							# test that outputs are equal for slice
  							self.parent.assertTrue(torch.allclose(lowerCamelCase					,			lowerCamelCase					,			atol=1E-3	)	)
  
  
  
  					def      __a    (		self    :    List[Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    str					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]	):
  							lowerCamelCase_  :    Dict   	=			RemBertForQuestionAnswering(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    List[Any]   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			start_positions=lowerCamelCase					,			end_positions=lowerCamelCase					,			)
  							self.parent.assertEqual(result.start_logits.shape					,			(self.batch_size, self.seq_length)	)
  							self.parent.assertEqual(result.end_logits.shape					,			(self.batch_size, self.seq_length)	)
  
  
  
  					def      __a    (		self    :    int					,			lowerCamelCase    :    str					,			lowerCamelCase    :    str					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.num_labels
  							lowerCamelCase_  :    Union[str, Any]   	=			RemBertForSequenceClassification(lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    Any   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
  							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_labels)	)
  
  
  
  					def      __a    (		self    :    Any					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[str]	):
  							lowerCamelCase_  :    Tuple   	=			self.num_labels
  							lowerCamelCase_  :    Tuple   	=			RemBertForTokenClassification(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    Optional[int]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
  							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.num_labels)	)
  
  
  
  					def      __a    (		self    :    str					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    Tuple	):
  							lowerCamelCase_  :    Any   	=			self.num_choices
  							lowerCamelCase_  :    Dict   	=			RemBertForMultipleChoice(config=lowerCamelCase	)
  							model.to(lowerCamelCase	)
  							model.eval()
  							lowerCamelCase_  :    Any   	=			input_ids.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
  							lowerCamelCase_  :    List[Any]   	=			token_type_ids.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
  							lowerCamelCase_  :    List[Any]   	=			input_mask.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
  							lowerCamelCase_  :    str   	=			model(
  							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase					,			)
  							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_choices)	)
  
  
  
  
  
  					def      __a    (		self    :    Optional[Any]	):
  							lowerCamelCase_  :    Dict   	=			self.prepare_config_and_inputs()
  							(
  							    lowerCamelCase_			
  							)  :    int   	=			config_and_inputs
  							lowerCamelCase_  :    Optional[Any]   	=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
  							return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class 	UpperCamelCase_     (     snake_case__     ,    snake_case__     ,    unittest.TestCase       ):
  					_a			:	int							=				(
  					    (
  					        RemBertModel,
  					        RemBertForMaskedLM,
  					        RemBertForCausalLM,
  					        RemBertForMultipleChoice,
  					        RemBertForQuestionAnswering,
  					        RemBertForSequenceClassification,
  					        RemBertForTokenClassification,
  					    )
  					    if is_torch_available()
  					    else ()
  					)
  					_a			:	Tuple							=				(RemBertForCausalLM,) if is_torch_available() else ()
  					_a			:	List[Any]							=				(
  					    {
  					        'feature-extraction': RemBertModel,
  					        'fill-mask': RemBertForMaskedLM,
  					        'question-answering': RemBertForQuestionAnswering,
  					        'text-classification': RemBertForSequenceClassification,
  					        'text-generation': RemBertForCausalLM,
  					        'token-classification': RemBertForTokenClassification,
  					        'zero-shot': RemBertForSequenceClassification,
  					    }
  					    if is_torch_available()
  					    else {}
  					)
  
  
  
  					def      __a    (		self    :    Dict	):
  							lowerCamelCase_  :    Dict   	=			RemBertModelTester(self	)
  							lowerCamelCase_  :    str   	=			ConfigTester(self					,			config_class=lowerCamelCase					,			hidden_size=37	)
  
  
  
  					def      __a    (		self    :    Optional[int]	):
  							self.config_tester.run_common_tests()
  
  
  
  					def      __a    (		self    :    List[Any]	):
  							lowerCamelCase_  :    List[str]   	=			self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_model(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    Any	):
  							lowerCamelCase_  :    Union[str, Any]   	=			self.model_tester.prepare_config_and_inputs()
  							for type in ["absolute", "relative_key", "relative_key_query"]:
  									lowerCamelCase_  :    Optional[Any]   	=			type
  									self.model_tester.create_and_check_model(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    Dict	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    int	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    str	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs_for_decoder()
  							self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    int	):
  							lowerCamelCase_  :    List[Any]   	=			self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_question_answering(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    str	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    int	):
  							lowerCamelCase_  :    Any   	=			self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_token_classification(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    List[str]	):
  							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs_for_decoder()
  							self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    str	):
  							# This regression test was failing with PyTorch < 1.3
  							(
  							    lowerCamelCase_			
  							)  :    int   	=			self.model_tester.prepare_config_and_inputs_for_decoder()
  
  							lowerCamelCase_  :    str   	=			None
  
  							self.model_tester.create_and_check_model_as_decoder(
  							    lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			)
  
  
  
  
  
  					@slow
  					def      __a    (		self    :    Any	):
  							for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  									lowerCamelCase_  :    int   	=			RemBertModel.from_pretrained(lowerCamelCase	)
  									self.assertIsNotNone(lowerCamelCase	)
  
  
  
  
  
  
  @require_torch
  class 	UpperCamelCase_     (     unittest.TestCase       ):
  
  
  
  					@slow
  					def      __a    (		self    :    Dict	):
  							# Test exact values at the last hidden layer
  							lowerCamelCase_  :    Dict   	=			RemBertModel.from_pretrained('google/rembert'	)
  							lowerCamelCase_  :    List[Any]   	=			torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]	)
  							lowerCamelCase_  :    Any   	=			torch.tensor([[0, 0, 0, 1, 1]]	)
  							with torch.no_grad():
  									lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase					,			token_type_ids=lowerCamelCase					,			output_hidden_states=lowerCamelCase	)
  
  							lowerCamelCase_  :    Union[str, Any]   	=			11_52
  
  							lowerCamelCase_  :    Optional[int]   	=			torch.Size((1, 5, hidden_size)	)
  							self.assertEqual(output['last_hidden_state'].shape					,			lowerCamelCase	)
  
  							lowerCamelCase_  :    Tuple   	=			torch.tensor(
  							    [
  							        [
  							            [0.0_754, -0.2_022, 0.1_904],
  							            [-0.3_354, -0.3_692, -0.4_791],
  							            [-0.2_314, -0.6_729, -0.0_749],
  							            [-0.0_396, -0.3_105, -0.4_234],
  							            [-0.1_571, -0.0_525, 0.5_353],
  							        ]
  							    ]	)
  
  							# Running on the original tf implementation gives slightly different results here.
  							# Not clear why this variations is present
  							# TODO: Find reason for discrepancy
  							# expected_original_implementation = [[
  							#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
  							#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
  							#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
  							#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
  							#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
  							# ]]
  
  							self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]					,			lowerCamelCase					,			atol=1E-4	)	)
  
style_context: 
  
  
  
  
  
  import gc
  import unittest
  
  import numpy as np
  import torch
  
  from diffusers import DDPMScheduler, MidiProcessor, SpectrogramDiffusionPipeline
  from diffusers.pipelines.spectrogram_diffusion import SpectrogramContEncoder, SpectrogramNotesEncoder, TaFilmDecoder
  from diffusers.utils import require_torch_gpu, skip_mps, slow, torch_device
  from diffusers.utils.testing_utils import enable_full_determinism, require_note_seq, require_onnxruntime
  
  from ..pipeline_params import TOKENS_TO_AUDIO_GENERATION_BATCH_PARAMS, TOKENS_TO_AUDIO_GENERATION_PARAMS
  from ..test_pipelines_common import PipelineTesterMixin
  
  
  enable_full_determinism()
  
  
  _lowercase		:     Optional[Any] 					="""./tests/fixtures/elise_format0.mid"""
  
  
  
  
  
  
  @unittest.skip('The note-seq package currently throws an error on import'       )
  class 	UpperCamelCase_     (     snake_case__     ,    unittest.TestCase       ):
  					_a			:	int							=				SpectrogramDiffusionPipeline
  					_a			:	Dict							=				PipelineTesterMixin.required_optional_params - {
  					    'callback',
  					    'latents',
  					    'callback_steps',
  					    'output_type',
  					    'num_images_per_prompt',
  					}
  					_a			:	str							=				False
  					_a			:	Union[str, Any]							=				False
  					_a			:	Optional[int]							=				TOKENS_TO_AUDIO_GENERATION_PARAMS
  					_a			:	Optional[Any]							=				TOKENS_TO_AUDIO_GENERATION_BATCH_PARAMS
  
  
  
  					def      __a    (		self    :    Dict	):
  							torch.manual_seed(0	)
  							lowerCamelCase_  :    List[str]   	=			SpectrogramNotesEncoder(
  							    max_length=20_48					,			vocab_size=15_36					,			d_model=7_68					,			dropout_rate=0.1					,			num_layers=1					,			num_heads=1					,			d_kv=4					,			d_ff=20_48					,			feed_forward_proj='gated-gelu'					,			)
  
  							lowerCamelCase_  :    Union[str, Any]   	=			SpectrogramContEncoder(
  							    input_dims=1_28					,			targets_context_length=2_56					,			d_model=7_68					,			dropout_rate=0.1					,			num_layers=1					,			num_heads=1					,			d_kv=4					,			d_ff=20_48					,			feed_forward_proj='gated-gelu'					,			)
  
  							lowerCamelCase_  :    Union[str, Any]   	=			TaFilmDecoder(
  							    input_dims=1_28					,			targets_length=2_56					,			max_decoder_noise_time=20_000.0					,			d_model=7_68					,			num_layers=1					,			num_heads=1					,			d_kv=4					,			d_ff=20_48					,			dropout_rate=0.1					,			)
  
  							lowerCamelCase_  :    Any   	=			DDPMScheduler()
  
  							lowerCamelCase_  :    str   	=			{
  							    'notes_encoder': notes_encoder.eval(),
  							    'continuous_encoder': continuous_encoder.eval(),
  							    'decoder': decoder.eval(),
  							    'scheduler': scheduler,
  							    'melgan': None,
  							}
  							return components
  
  
  
  					def      __a    (		self    :    Any					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    int=0	):
  							if str(lowerCamelCase	).startswith('mps'	):
  									lowerCamelCase_  :    Optional[int]   	=			torch.manual_seed(lowerCamelCase	)
  							else:
  									lowerCamelCase_  :    Dict   	=			torch.Generator(device=lowerCamelCase	).manual_seed(lowerCamelCase	)
  							lowerCamelCase_  :    str   	=			{
  							    'input_tokens': [
  							        [11_34, 90, 11_35, 11_33, 10_80, 1_12, 11_32, 10_80, 11_33, 10_79, 1_33, 11_32, 10_79, 11_33, 1] + [0] * 20_33
  							    ],
  							    'generator': generator,
  							    'num_inference_steps': 4,
  							    'output_type': 'mel',
  							}
  							return inputs
  
  
  
  					def      __a    (		self    :    Dict	):
  							lowerCamelCase_  :    str   	=			'cpu'  # ensure determinism for the device-dependent torch.Generator
  							lowerCamelCase_  :    Dict   	=			self.get_dummy_components()
  							lowerCamelCase_  :    Tuple   	=			SpectrogramDiffusionPipeline(**lowerCamelCase	)
  							lowerCamelCase_  :    List[str]   	=			pipe.to(lowerCamelCase	)
  							pipe.set_progress_bar_config(disable=lowerCamelCase	)
  
  							lowerCamelCase_  :    Optional[Any]   	=			self.get_dummy_inputs(lowerCamelCase	)
  							lowerCamelCase_  :    str   	=			pipe(**lowerCamelCase	)
  							lowerCamelCase_  :    Tuple   	=			output.audios
  
  							lowerCamelCase_  :    Tuple   	=			mel[0, -3:, -3:]
  
  							assert mel_slice.shape == (3, 3)
  							lowerCamelCase_  :    List[str]   	=			np.array(
  							    [-11.512_925, -4.788_215, -0.46_172_905, -2.051_715, -10.539_147, -10.970_963, -9.091_634, 4.0, 4.0]	)
  							assert np.abs(mel_slice.flatten() - expected_slice	).max() < 1E-2
  
  
  
  					@skip_mps
  					def      __a    (		self    :    Any	):
  							return super().test_save_load_local()
  
  
  
  					@skip_mps
  					def      __a    (		self    :    Tuple	):
  							return super().test_dict_tuple_outputs_equivalent()
  
  
  
  					@skip_mps
  					def      __a    (		self    :    Dict	):
  							return super().test_save_load_optional_components()
  
  
  
  					@skip_mps
  					def      __a    (		self    :    str	):
  							return super().test_attention_slicing_forward_pass()
  
  
  
  					def      __a    (		self    :    Any	):
  							pass
  
  
  
  					def      __a    (		self    :    Optional[int]	):
  							pass
  
  
  
  
  
  					@skip_mps
  					def      __a    (		self    :    Optional[Any]	):
  							return super().test_progress_bar()
  
  
  
  
  
  
  @slow
  @require_torch_gpu
  @require_onnxruntime
  @require_note_seq
  class 	UpperCamelCase_     (     unittest.TestCase       ):
  
  
  
  					def      __a    (		self    :    Optional[int]	):
  							# clean up the VRAM after each test
  							super().tearDown()
  							gc.collect()
  							torch.cuda.empty_cache()
  
  
  
  					def      __a    (		self    :    str	):
  							# TODO - test that pipeline can decode tokens in a callback
  							# so that music can be played live
  							lowerCamelCase_  :    Optional[Any]   	=			torch_device
  
  							lowerCamelCase_  :    Union[str, Any]   	=			SpectrogramDiffusionPipeline.from_pretrained('google/music-spectrogram-diffusion'	)
  							lowerCamelCase_  :    str   	=			pipe.melgan
  							lowerCamelCase_  :    Optional[int]   	=			None
  
  							lowerCamelCase_  :    str   	=			pipe.to(lowerCamelCase	)
  							pipe.set_progress_bar_config(disable=lowerCamelCase	)
  
  							def callback(lowerCamelCase    :    int					,			lowerCamelCase    :    int	):
  									# decode mel to audio
  									lowerCamelCase_  :    Union[str, Any]   	=			melgan(input_features=mel_output.astype(np.floataa	)	)[0]
  									assert len(audio[0]	) == 8_19_20 * (step + 1)
  									# simulate that audio is played
  									return audio
  
  							lowerCamelCase_  :    Any   	=			MidiProcessor()
  							lowerCamelCase_  :    str   	=			processor(lowerCamelCase	)
  
  							lowerCamelCase_  :    str   	=			input_tokens[:3]
  							lowerCamelCase_  :    Dict   	=			torch.manual_seed(0	)
  							pipe(lowerCamelCase					,			num_inference_steps=5					,			generator=lowerCamelCase					,			callback=lowerCamelCase					,			output_type='mel'	)
  
  
  
  					def      __a    (		self    :    str	):
  							lowerCamelCase_  :    List[str]   	=			torch_device
  
  							lowerCamelCase_  :    List[str]   	=			SpectrogramDiffusionPipeline.from_pretrained('google/music-spectrogram-diffusion'	)
  							lowerCamelCase_  :    Union[str, Any]   	=			pipe.to(lowerCamelCase	)
  							pipe.set_progress_bar_config(disable=lowerCamelCase	)
  							lowerCamelCase_  :    int   	=			MidiProcessor()
  
  							lowerCamelCase_  :    Dict   	=			processor(lowerCamelCase	)
  							# just run two denoising loops
  							lowerCamelCase_  :    Optional[Any]   	=			input_tokens[:2]
  
  							lowerCamelCase_  :    List[Any]   	=			torch.manual_seed(0	)
  							lowerCamelCase_  :    str   	=			pipe(lowerCamelCase					,			num_inference_steps=2					,			generator=lowerCamelCase	)
  
  							lowerCamelCase_  :    Dict   	=			output.audios[0]
  
  							assert abs(np.abs(lowerCamelCase	).sum() - 3_612.841	) < 1E-1
  
  
  
  
  
  					def      __a    (		self    :    List[Any]	):
  							lowerCamelCase_  :    int   	=			torch_device
  
  							lowerCamelCase_  :    str   	=			SpectrogramDiffusionPipeline.from_pretrained('google/music-spectrogram-diffusion'	)
  							lowerCamelCase_  :    Optional[int]   	=			pipe.to(lowerCamelCase	)
  							pipe.set_progress_bar_config(disable=lowerCamelCase	)
  
  							lowerCamelCase_  :    Tuple   	=			MidiProcessor()
  
  							lowerCamelCase_  :    int   	=			processor(lowerCamelCase	)
  
  							# just run 4 denoising loops
  							lowerCamelCase_  :    Optional[Any]   	=			input_tokens[:4]
  
  							lowerCamelCase_  :    List[Any]   	=			torch.manual_seed(0	)
  							lowerCamelCase_  :    int   	=			pipe(lowerCamelCase					,			num_inference_steps=1_00					,			generator=lowerCamelCase	)
  
  							lowerCamelCase_  :    List[Any]   	=			output.audios[0]
  							assert abs(np.abs(lowerCamelCase	).sum() - 9_389.1_111	) < 5E-2
  