code: 
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  						import torch
  
  						from transformers import (
  						    RemBertConfig,
  						    RemBertForCausalLM,
  						    RemBertForMaskedLM,
  						    RemBertForMultipleChoice,
  						    RemBertForQuestionAnswering,
  						    RemBertForSequenceClassification,
  						    RemBertForTokenClassification,
  						    RemBertModel,
  						)
  						from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  class      UpperCamelCase_			:
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						def __init__( self       :int		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Optional[int]=13		,						lowerCAmelCase__       :Optional[int]=7		,						lowerCAmelCase__       :Tuple=True		,						lowerCAmelCase__       :Optional[int]=True		,						lowerCAmelCase__       :Optional[int]=True		,						lowerCAmelCase__       :Dict=True		,						lowerCAmelCase__       :Any=99		,						lowerCAmelCase__       :int=32		,						lowerCAmelCase__       :Any=18		,						lowerCAmelCase__       :List[Any]=43		,						lowerCAmelCase__       :int=5		,						lowerCAmelCase__       :Union[str, Any]=4		,						lowerCAmelCase__       :Optional[Any]=37		,						lowerCAmelCase__       :List[Any]="gelu"		,						lowerCAmelCase__       :List[str]=0.1		,						lowerCAmelCase__       :Dict=0.1		,						lowerCAmelCase__       :Tuple=512		,						lowerCAmelCase__       :Tuple=16		,						lowerCAmelCase__       :Union[str, Any]=2		,						lowerCAmelCase__       :str=0.02		,						lowerCAmelCase__       :str=3		,						lowerCAmelCase__       :Union[str, Any]=4		,						lowerCAmelCase__       :Tuple=None		,						)     ->int:
  										lowercase						    =			parent
  										lowercase						    =			batch_size
  										lowercase						    =			seq_length
  										lowercase						    =			is_training
  										lowercase						    =			use_input_mask
  										lowercase						    =			use_token_type_ids
  										lowercase						    =			use_labels
  										lowercase						    =			vocab_size
  										lowercase						    =			hidden_size
  										lowercase						    =			input_embedding_size
  										lowercase						    =			output_embedding_size
  										lowercase						    =			num_hidden_layers
  										lowercase						    =			num_attention_heads
  										lowercase						    =			intermediate_size
  										lowercase						    =			hidden_act
  										lowercase						    =			hidden_dropout_prob
  										lowercase						    =			attention_probs_dropout_prob
  										lowercase						    =			max_position_embeddings
  										lowercase						    =			type_vocab_size
  										lowercase						    =			type_sequence_label_size
  										lowercase						    =			initializer_range
  										lowercase						    =			num_labels
  										lowercase						    =			num_choices
  										lowercase						    =			scope
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[str]    )     ->List[Any]:
  										lowercase						    =			ids_tensor([self.batch_size, self.seq_length]		,						self.vocab_size    )
  
  										lowercase						    =			None
  										if self.use_input_mask:
  														lowercase						    =			random_attention_mask([self.batch_size, self.seq_length]    )
  
  										lowercase						    =			None
  										if self.use_token_type_ids:
  														lowercase						    =			ids_tensor([self.batch_size, self.seq_length]		,						self.type_vocab_size    )
  
  										lowercase						    =			None
  										lowercase						    =			None
  										lowercase						    =			None
  										if self.use_labels:
  														lowercase						    =			ids_tensor([self.batch_size]		,						self.type_sequence_label_size    )
  														lowercase						    =			ids_tensor([self.batch_size, self.seq_length]		,						self.num_labels    )
  														lowercase						    =			ids_tensor([self.batch_size]		,						self.num_choices    )
  
  										lowercase						    =			RemBertConfig(
  										    vocab_size=self.vocab_size		,						hidden_size=self.hidden_size		,						input_embedding_size=self.input_embedding_size		,						output_embedding_size=self.output_embedding_size		,						num_hidden_layers=self.num_hidden_layers		,						num_attention_heads=self.num_attention_heads		,						intermediate_size=self.intermediate_size		,						hidden_act=self.hidden_act		,						hidden_dropout_prob=self.hidden_dropout_prob		,						attention_probs_dropout_prob=self.attention_probs_dropout_prob		,						max_position_embeddings=self.max_position_embeddings		,						type_vocab_size=self.type_vocab_size		,						is_decoder=lowerCAmelCase__		,						initializer_range=self.initializer_range		,						)
  
  										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Any    )     ->List[Any]:
  										(
  										    (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  
  										)						    =			self.prepare_config_and_inputs()
  
  										lowercase						    =			True
  										lowercase						    =			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
  										lowercase						    =			ids_tensor([self.batch_size, self.seq_length]		,						vocab_size=2    )
  
  										return (
  										    config,
  										    input_ids,
  										    token_type_ids,
  										    input_mask,
  										    sequence_labels,
  										    token_labels,
  										    choice_labels,
  										    encoder_hidden_states,
  										    encoder_attention_mask,
  										)
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :Tuple    )     ->str:
  										lowercase						    =			RemBertModel(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__    )
  										lowercase						    =			model(lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__    )
  										lowercase						    =			model(lowerCAmelCase__    )
  										self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]		,						lowerCAmelCase__       :int		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :int		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Tuple		,						)     ->str:
  										lowercase						    =			True
  										lowercase						    =			RemBertModel(lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						encoder_hidden_states=lowerCAmelCase__		,						encoder_attention_mask=lowerCAmelCase__		,						)
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						encoder_hidden_states=lowerCAmelCase__		,						)
  										lowercase						    =			model(lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__    )
  										self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Dict		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Optional[int]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :Optional[int]		,						)     ->List[Any]:
  										lowercase						    =			RemBertForCausalLM(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						labels=lowerCAmelCase__    )
  										self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.vocab_size)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Any		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :List[Any]		,						lowerCAmelCase__       :int		,						lowerCAmelCase__       :Optional[int]		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :List[str]    )     ->Optional[Any]:
  										lowercase						    =			RemBertForMaskedLM(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						labels=lowerCAmelCase__    )
  										self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.vocab_size)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :List[Any]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Dict		,						)     ->str:
  										lowercase						    =			True
  										lowercase						    =			True
  										lowercase						    =			RemBertForCausalLM(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  
  										# first forward pass
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						encoder_hidden_states=lowerCAmelCase__		,						encoder_attention_mask=lowerCAmelCase__		,						use_cache=lowerCAmelCase__		,						)
  										lowercase						    =			outputs.past_key_values
  
  										# create hypothetical multiple next token and extent to next_input_ids
  										lowercase						    =			ids_tensor((self.batch_size, 3)		,						config.vocab_size    )
  										lowercase						    =			ids_tensor((self.batch_size, 3)		,						vocab_size=2    )
  
  										# append to next input_ids and
  										lowercase						    =			torch.cat([input_ids, next_tokens]		,						dim=-1    )
  										lowercase						    =			torch.cat([input_mask, next_mask]		,						dim=-1    )
  
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						encoder_hidden_states=lowerCAmelCase__		,						encoder_attention_mask=lowerCAmelCase__		,						output_hidden_states=lowerCAmelCase__		,						)["hidden_states"][0]
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						encoder_hidden_states=lowerCAmelCase__		,						encoder_attention_mask=lowerCAmelCase__		,						past_key_values=lowerCAmelCase__		,						output_hidden_states=lowerCAmelCase__		,						)["hidden_states"][0]
  
  										# select random slice
  										lowercase						    =			ids_tensor((1,)		,						output_from_past.shape[-1]    ).item()
  										lowercase						    =			output_from_no_past[:, -3:, random_slice_idx].detach()
  										lowercase						    =			output_from_past[:, :, random_slice_idx].detach()
  
  										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )
  
  										# test that outputs are equal for slice
  										self.parent.assertTrue(torch.allclose(lowerCAmelCase__		,						lowerCAmelCase__		,						atol=1E-3    )    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Dict		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Dict    )     ->List[Any]:
  										lowercase						    =			RemBertForQuestionAnswering(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						start_positions=lowerCAmelCase__		,						end_positions=lowerCAmelCase__		,						)
  										self.parent.assertEqual(result.start_logits.shape		,						(self.batch_size, self.seq_length)    )
  										self.parent.assertEqual(result.end_logits.shape		,						(self.batch_size, self.seq_length)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Any		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :Dict    )     ->List[str]:
  										lowercase						    =			self.num_labels
  										lowercase						    =			RemBertForSequenceClassification(lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						labels=lowerCAmelCase__    )
  										self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_labels)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[str]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Optional[int]		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :List[str]    )     ->List[str]:
  										lowercase						    =			self.num_labels
  										lowercase						    =			RemBertForTokenClassification(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			model(lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						labels=lowerCAmelCase__    )
  										self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.num_labels)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Optional[int]		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :Tuple    )     ->Tuple:
  										lowercase						    =			self.num_choices
  										lowercase						    =			RemBertForMultipleChoice(config=lowerCAmelCase__    )
  										model.to(lowerCAmelCase__    )
  										model.eval()
  										lowercase						    =			input_ids.unsqueeze(1    ).expand(-1		,						self.num_choices		,						-1    ).contiguous()
  										lowercase						    =			token_type_ids.unsqueeze(1    ).expand(-1		,						self.num_choices		,						-1    ).contiguous()
  										lowercase						    =			input_mask.unsqueeze(1    ).expand(-1		,						self.num_choices		,						-1    ).contiguous()
  										lowercase						    =			model(
  										    lowerCAmelCase__		,						attention_mask=lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						labels=lowerCAmelCase__		,						)
  										self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_choices)    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Optional[int]    )     ->Any:
  										lowercase						    =			self.prepare_config_and_inputs()
  										(
  										    (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  
  										)						    =			config_and_inputs
  										lowercase						    =			{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
  										return config, inputs_dict
  
  
  
  
  
  @require_torch
  class      UpperCamelCase_			(			__a		,						__a		,						unittest.TestCase		):
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						UpperCamelCase			:     int							      =				(
  						    (
  						        RemBertModel,
  						        RemBertForMaskedLM,
  						        RemBertForCausalLM,
  						        RemBertForMultipleChoice,
  						        RemBertForQuestionAnswering,
  						        RemBertForSequenceClassification,
  						        RemBertForTokenClassification,
  						    )
  						    if is_torch_available()
  						    else ()
  						)
  						UpperCamelCase			:     Optional[Any]							      =				(RemBertForCausalLM,) if is_torch_available() else ()
  						UpperCamelCase			:     int							      =				(
  						    {
  						        '''feature-extraction''': RemBertModel,
  						        '''fill-mask''': RemBertForMaskedLM,
  						        '''question-answering''': RemBertForQuestionAnswering,
  						        '''text-classification''': RemBertForSequenceClassification,
  						        '''text-generation''': RemBertForCausalLM,
  						        '''token-classification''': RemBertForTokenClassification,
  						        '''zero-shot''': RemBertForSequenceClassification,
  						    }
  						    if is_torch_available()
  						    else {}
  						)
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]    )     ->str:
  										lowercase						    =			RemBertModelTester(self    )
  										lowercase						    =			ConfigTester(self		,						config_class=lowerCAmelCase__		,						hidden_size=37    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :int    )     ->Dict:
  										self.config_tester.run_common_tests()
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]    )     ->Tuple:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										self.model_tester.create_and_check_model(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Optional[Any]    )     ->List[str]:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										for type in ["absolute", "relative_key", "relative_key_query"]:
  														lowercase						    =			type
  														self.model_tester.create_and_check_model(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Any    )     ->Union[str, Any]:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Optional[Any]    )     ->Any:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]    )     ->str:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs_for_decoder()
  										self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :str    )     ->Tuple:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :int    )     ->Union[str, Any]:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Optional[Any]    )     ->str:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs()
  										self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Tuple    )     ->str:
  										lowercase						    =			self.model_tester.prepare_config_and_inputs_for_decoder()
  										self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]    )     ->List[str]:
  										# This regression test was failing with PyTorch < 1.3
  										(
  										    (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  (
  										    lowercase
  										)    ,  
  										)						    =			self.model_tester.prepare_config_and_inputs_for_decoder()
  
  										lowercase						    =			None
  
  										self.model_tester.create_and_check_model_as_decoder(
  										    lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						lowerCAmelCase__		,						)
  
  
  
  
  
  						@slow
  						def 					SCREAMING_SNAKE_CASE( self       :Tuple    )     ->Dict:
  										for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  														lowercase						    =			RemBertModel.from_pretrained(lowerCAmelCase__    )
  														self.assertIsNotNone(lowerCAmelCase__    )
  
  
  
  
  
  @require_torch
  class      UpperCamelCase_			(			unittest.TestCase		):
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						@slow
  						def 					SCREAMING_SNAKE_CASE( self       :Optional[int]    )     ->Optional[int]:
  										# Test exact values at the last hidden layer
  										lowercase						    =			RemBertModel.from_pretrained("google/rembert"    )
  										lowercase						    =			torch.tensor([[312, 56498, 313, 2125, 313]]    )
  										lowercase						    =			torch.tensor([[0, 0, 0, 1, 1]]    )
  										with torch.no_grad():
  														lowercase						    =			model(lowerCAmelCase__		,						token_type_ids=lowerCAmelCase__		,						output_hidden_states=lowerCAmelCase__    )
  
  										lowercase						    =			1152
  
  										lowercase						    =			torch.Size((1, 5, hidden_size)    )
  										self.assertEqual(output["last_hidden_state"].shape		,						lowerCAmelCase__    )
  
  										lowercase						    =			torch.tensor(
  										    [
  										        [
  										            [0.07_54, -0.20_22, 0.19_04],
  										            [-0.33_54, -0.36_92, -0.47_91],
  										            [-0.23_14, -0.67_29, -0.07_49],
  										            [-0.03_96, -0.31_05, -0.42_34],
  										            [-0.15_71, -0.05_25, 0.53_53],
  										        ]
  										    ]    )
  
  										# Running on the original tf implementation gives slightly different results here.
  										# Not clear why this variations is present
  										# TODO: Find reason for discrepancy
  										# expected_original_implementation = [[
  										#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
  										#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
  										#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
  										#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
  										#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
  										# ]]
  
  										self.assertTrue(torch.allclose(output["last_hidden_state"][:, :, :3]		,						lowerCAmelCase__		,						atol=1E-4    )    )
  
style_context: 
  from functools import partial
  from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload
  
  import torch
  import torch.nn as nn
  import torch.types
  
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				if not inplace:
  								lowercase						    =			ma + ma
  				else:
  								ma += ma
  
  				return ma
  
  def    __snake_case       (  __magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			-1 * len(__magic_name__	)
  				lowercase						    =			list(range(len(tensor.shape[:zero_index]	)	)	)
  				return tensor.permute(first_inds + [zero_index + i for i in inds]	)
  
  def    __snake_case       (  __magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				return t.reshape(t.shape[:-no_dims] + (-1,)	)
  
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__		,						__magic_name__ = 1e-4	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			mask.expand(*value.shape	)
  				return torch.sum(mask * value		,						dim=__magic_name__	) / (eps + torch.sum(__magic_name__		,						dim=__magic_name__	))
  
  def    __snake_case       (  __magic_name__		,						__magic_name__ = 2.3_1_2_5		,						__magic_name__ = 2_1.6_8_7_5		,						__magic_name__ = 64	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			torch.linspace(__magic_name__		,						__magic_name__		,						no_bins - 1		,						device=pts.device	)
  				lowercase						    =			torch.sqrt(torch.sum((pts.unsqueeze(-2	) - pts.unsqueeze(-3	)) ** 2		,						dim=-1	)	)
  				return torch.bucketize(__magic_name__		,						__magic_name__	)
  
  def    __snake_case       (  __magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			dicts[0]
  				lowercase						    =			{}
  				for k, v in first.items():
  								lowercase						    =			[d[k] for d in dicts]
  								if isinstance(__magic_name__		,						__magic_name__	):
  												lowercase						    =			dict_multimap(__magic_name__		,						__magic_name__	)
  								else:
  												lowercase						    =			fn(__magic_name__	)
  
  				return new_dict
  
  def    __snake_case       (  __magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			v_bins.view(((1,) * len(x.shape	)) + (len(__magic_name__	),)	)
  				lowercase						    =			x[..., None] - reshaped_bins
  				lowercase						    =			torch.argmin(torch.abs(__magic_name__	)		,						dim=-1	)
  				return nn.functional.one_hot(__magic_name__		,						num_classes=len(__magic_name__	)	).float()
  
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__ = 0		,						__magic_name__ = 0	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			[]
  				for i, s in enumerate(data.shape[:no_batch_dims]	):
  								lowercase						    =			torch.arange(__magic_name__	)
  								lowercase						    =			r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape	) - i - 1)))	)
  								ranges.append(__magic_name__	)
  
  				lowercase						    =			[slice(__magic_name__	) for _ in range(len(data.shape	) - no_batch_dims	)]
  				lowercase						    =			inds
  				ranges.extend(__magic_name__	)
  				# Matt note: Editing this to get around the behaviour of using a list as an array index changing
  				# in recent Numpy versions
  				return data[tuple(__magic_name__	)]
  
  
  _snake_case			:       List[Any]      =						TypeVar("T")
  
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			{}
  				for k, v in dic.items():
  								if isinstance(__magic_name__		,						__magic_name__	):
  												lowercase						    =			dict_map(__magic_name__		,						__magic_name__		,						__magic_name__	)
  								else:
  												lowercase						    =			tree_map(__magic_name__		,						__magic_name__		,						__magic_name__	)
  
  				return new_dict
  
  @overload
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				...
  
  @overload
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				...
  
  @overload
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				...
  
  @overload
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				...
  
  def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				if isinstance(__magic_name__		,						__magic_name__	):
  								return dict_map(__magic_name__		,						__magic_name__		,						__magic_name__	)
  				elif isinstance(__magic_name__		,						__magic_name__	):
  								return [tree_map(__magic_name__		,						__magic_name__		,						__magic_name__	) for x in tree]
  				elif isinstance(__magic_name__		,						__magic_name__	):
  								return tuple(tree_map(__magic_name__		,						__magic_name__		,						__magic_name__	) for x in tree	)
  				elif isinstance(__magic_name__		,						__magic_name__	):
  								return fn(__magic_name__	)
  				else:
  								print(type(__magic_name__	)	)
  								raise ValueError("Not supported"	)
  
  
  
  
  _snake_case			:       Union[str, Any]      =						partial(tree_map, leaf_type=torch.Tensor)
  