code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  			import torch
  
  			from transformers import (
  			    RemBertConfig,
  			    RemBertForCausalLM,
  			    RemBertForMaskedLM,
  			    RemBertForMultipleChoice,
  			    RemBertForQuestionAnswering,
  			    RemBertForSequenceClassification,
  			    RemBertForTokenClassification,
  			    RemBertModel,
  			)
  			from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class 							__snake_case    :
  
  
  
  
  
  	def __init__(		self			,					UpperCamelCase_			,					UpperCamelCase_=13			,					UpperCamelCase_=7			,					UpperCamelCase_=True			,					UpperCamelCase_=True			,					UpperCamelCase_=True			,					UpperCamelCase_=True			,					UpperCamelCase_=99			,					UpperCamelCase_=32			,					UpperCamelCase_=18			,					UpperCamelCase_=43			,					UpperCamelCase_=5			,					UpperCamelCase_=4			,					UpperCamelCase_=37			,					UpperCamelCase_="gelu"			,					UpperCamelCase_=0.1			,					UpperCamelCase_=0.1			,					UpperCamelCase_=512			,					UpperCamelCase_=16			,					UpperCamelCase_=2			,					UpperCamelCase_=0.0_2			,					UpperCamelCase_=3			,					UpperCamelCase_=4			,					UpperCamelCase_=None			,					)			->		Any:
  		snake_case__  					=      parent
  		snake_case__  					=      batch_size
  		snake_case__  					=      seq_length
  		snake_case__  					=      is_training
  		snake_case__  					=      use_input_mask
  		snake_case__  					=      use_token_type_ids
  		snake_case__  					=      use_labels
  		snake_case__  					=      vocab_size
  		snake_case__  					=      hidden_size
  		snake_case__  					=      input_embedding_size
  		snake_case__  					=      output_embedding_size
  		snake_case__  					=      num_hidden_layers
  		snake_case__  					=      num_attention_heads
  		snake_case__  					=      intermediate_size
  		snake_case__  					=      hidden_act
  		snake_case__  					=      hidden_dropout_prob
  		snake_case__  					=      attention_probs_dropout_prob
  		snake_case__  					=      max_position_embeddings
  		snake_case__  					=      type_vocab_size
  		snake_case__  					=      type_sequence_label_size
  		snake_case__  					=      initializer_range
  		snake_case__  					=      num_labels
  		snake_case__  					=      num_choices
  		snake_case__  					=      scope
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Any:
  		snake_case__  					=      ids_tensor([self.batch_size, self.seq_length]			,					self.vocab_size		)
  
  		snake_case__  					=      None
  		if self.use_input_mask:
  			snake_case__  					=      random_attention_mask([self.batch_size, self.seq_length]		)
  
  		snake_case__  					=      None
  		if self.use_token_type_ids:
  			snake_case__  					=      ids_tensor([self.batch_size, self.seq_length]			,					self.type_vocab_size		)
  
  		snake_case__  					=      None
  		snake_case__  					=      None
  		snake_case__  					=      None
  		if self.use_labels:
  			snake_case__  					=      ids_tensor([self.batch_size]			,					self.type_sequence_label_size		)
  			snake_case__  					=      ids_tensor([self.batch_size, self.seq_length]			,					self.num_labels		)
  			snake_case__  					=      ids_tensor([self.batch_size]			,					self.num_choices		)
  
  		snake_case__  					=      RemBertConfig(
  		    vocab_size=self.vocab_size			,					hidden_size=self.hidden_size			,					input_embedding_size=self.input_embedding_size			,					output_embedding_size=self.output_embedding_size			,					num_hidden_layers=self.num_hidden_layers			,					num_attention_heads=self.num_attention_heads			,					intermediate_size=self.intermediate_size			,					hidden_act=self.hidden_act			,					hidden_dropout_prob=self.hidden_dropout_prob			,					attention_probs_dropout_prob=self.attention_probs_dropout_prob			,					max_position_embeddings=self.max_position_embeddings			,					type_vocab_size=self.type_vocab_size			,					is_decoder=UpperCamelCase_			,					initializer_range=self.initializer_range			,					)
  
  		return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		(
  		    (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     
  		)  					=      self.prepare_config_and_inputs()
  
  		snake_case__  					=      True
  		snake_case__  					=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
  		snake_case__  					=      ids_tensor([self.batch_size, self.seq_length]			,					vocab_size=2		)
  
  		return (
  		    config,
  		    input_ids,
  		    token_type_ids,
  		    input_mask,
  		    sequence_labels,
  		    token_labels,
  		    choice_labels,
  		    encoder_hidden_states,
  		    encoder_attention_mask,
  		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Optional[int]:
  		snake_case__  					=      RemBertModel(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_		)
  		snake_case__  					=      model(UpperCamelCase_			,					token_type_ids=UpperCamelCase_		)
  		snake_case__  					=      model(UpperCamelCase_		)
  		self.parent.assertEqual(result.last_hidden_state.shape			,					(self.batch_size, self.seq_length, self.hidden_size)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					)			->		int:
  		snake_case__  					=      True
  		snake_case__  					=      RemBertModel(UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					encoder_hidden_states=UpperCamelCase_			,					encoder_attention_mask=UpperCamelCase_			,					)
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					encoder_hidden_states=UpperCamelCase_			,					)
  		snake_case__  					=      model(UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_		)
  		self.parent.assertEqual(result.last_hidden_state.shape			,					(self.batch_size, self.seq_length, self.hidden_size)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					)			->		Any:
  		snake_case__  					=      RemBertForCausalLM(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					labels=UpperCamelCase_		)
  		self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.vocab_size)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Optional[Any]:
  		snake_case__  					=      RemBertForMaskedLM(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					labels=UpperCamelCase_		)
  		self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.vocab_size)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					)			->		Optional[int]:
  		snake_case__  					=      True
  		snake_case__  					=      True
  		snake_case__  					=      RemBertForCausalLM(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  
  		# first forward pass
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					encoder_hidden_states=UpperCamelCase_			,					encoder_attention_mask=UpperCamelCase_			,					use_cache=UpperCamelCase_			,					)
  		snake_case__  					=      outputs.past_key_values
  
  		# create hypothetical multiple next token and extent to next_input_ids
  		snake_case__  					=      ids_tensor((self.batch_size, 3)			,					config.vocab_size		)
  		snake_case__  					=      ids_tensor((self.batch_size, 3)			,					vocab_size=2		)
  
  		# append to next input_ids and
  		snake_case__  					=      torch.cat([input_ids, next_tokens]			,					dim=-1		)
  		snake_case__  					=      torch.cat([input_mask, next_mask]			,					dim=-1		)
  
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					encoder_hidden_states=UpperCamelCase_			,					encoder_attention_mask=UpperCamelCase_			,					output_hidden_states=UpperCamelCase_			,					)['hidden_states'][0]
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					encoder_hidden_states=UpperCamelCase_			,					encoder_attention_mask=UpperCamelCase_			,					past_key_values=UpperCamelCase_			,					output_hidden_states=UpperCamelCase_			,					)['hidden_states'][0]
  
  		# select random slice
  		snake_case__  					=      ids_tensor((1,)			,					output_from_past.shape[-1]		).item()
  		snake_case__  					=      output_from_no_past[:, -3:, random_slice_idx].detach()
  		snake_case__  					=      output_from_past[:, :, random_slice_idx].detach()
  
  		self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)
  
  		# test that outputs are equal for slice
  		self.parent.assertTrue(torch.allclose(UpperCamelCase_			,					UpperCamelCase_			,					atol=1E-3		)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Optional[int]:
  		snake_case__  					=      RemBertForQuestionAnswering(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					start_positions=UpperCamelCase_			,					end_positions=UpperCamelCase_			,					)
  		self.parent.assertEqual(result.start_logits.shape			,					(self.batch_size, self.seq_length)		)
  		self.parent.assertEqual(result.end_logits.shape			,					(self.batch_size, self.seq_length)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Union[str, Any]:
  		snake_case__  					=      self.num_labels
  		snake_case__  					=      RemBertForSequenceClassification(UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					labels=UpperCamelCase_		)
  		self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.num_labels)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Any:
  		snake_case__  					=      self.num_labels
  		snake_case__  					=      RemBertForTokenClassification(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      model(UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					labels=UpperCamelCase_		)
  		self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.num_labels)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_		)			->		Optional[Any]:
  		snake_case__  					=      self.num_choices
  		snake_case__  					=      RemBertForMultipleChoice(config=UpperCamelCase_		)
  		model.to(UpperCamelCase_		)
  		model.eval()
  		snake_case__  					=      input_ids.unsqueeze(1		).expand(-1			,					self.num_choices			,					-1		).contiguous()
  		snake_case__  					=      token_type_ids.unsqueeze(1		).expand(-1			,					self.num_choices			,					-1		).contiguous()
  		snake_case__  					=      input_mask.unsqueeze(1		).expand(-1			,					self.num_choices			,					-1		).contiguous()
  		snake_case__  					=      model(
  		    UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					labels=UpperCamelCase_			,					)
  		self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.num_choices)		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      self.prepare_config_and_inputs()
  		(
  		    (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     
  		)  					=      config_and_inputs
  		snake_case__  					=      {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
  		return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class 							__snake_case    (				__magic_name__					,					__magic_name__					,					unittest.TestCase       ):
  	__lowerCAmelCase						=     (
  	    (
  	        RemBertModel,
  	        RemBertForMaskedLM,
  	        RemBertForCausalLM,
  	        RemBertForMultipleChoice,
  	        RemBertForQuestionAnswering,
  	        RemBertForSequenceClassification,
  	        RemBertForTokenClassification,
  	    )
  	    if is_torch_available()
  	    else ()
  	)
  	__lowerCAmelCase						=     (RemBertForCausalLM,) if is_torch_available() else ()
  	__lowerCAmelCase						=     (
  	    {
  	        '''feature-extraction''': RemBertModel,
  	        '''fill-mask''': RemBertForMaskedLM,
  	        '''question-answering''': RemBertForQuestionAnswering,
  	        '''text-classification''': RemBertForSequenceClassification,
  	        '''text-generation''': RemBertForCausalLM,
  	        '''token-classification''': RemBertForTokenClassification,
  	        '''zero-shot''': RemBertForSequenceClassification,
  	    }
  	    if is_torch_available()
  	    else {}
  	)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		snake_case__  					=      RemBertModelTester(self		)
  		snake_case__  					=      ConfigTester(self			,					config_class=UpperCamelCase_			,					hidden_size=37		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[Any]:
  		self.config_tester.run_common_tests()
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		self.model_tester.create_and_check_model(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		int:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		for type in ["absolute", "relative_key", "relative_key_query"]:
  			snake_case__  					=      type
  			self.model_tester.create_and_check_model(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		self.model_tester.create_and_check_for_masked_lm(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		str:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		self.model_tester.create_and_check_for_multiple_choice(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		str:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs_for_decoder()
  		self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		str:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		self.model_tester.create_and_check_for_question_answering(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs()
  		self.model_tester.create_and_check_for_token_classification(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		int:
  		snake_case__  					=      self.model_tester.prepare_config_and_inputs_for_decoder()
  		self.model_tester.create_and_check_model_as_decoder(*UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		# This regression test was failing with PyTorch < 1.3
  		(
  		    (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     (
  		    snake_case__
  		)					,     
  		)  					=      self.model_tester.prepare_config_and_inputs_for_decoder()
  
  		snake_case__  					=      None
  
  		self.model_tester.create_and_check_model_as_decoder(
  		    UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					UpperCamelCase_			,					)
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		Any:
  		for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  			snake_case__  					=      RemBertModel.from_pretrained(UpperCamelCase_		)
  			self.assertIsNotNone(UpperCamelCase_		)
  
  
  
  
  
  
  @require_torch
  class 							__snake_case    (				unittest.TestCase       ):
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		List[Any]:
  		# Test exact values at the last hidden layer
  		snake_case__  					=      RemBertModel.from_pretrained('google/rembert'		)
  		snake_case__  					=      torch.tensor([[312, 5_6498, 313, 2125, 313]]		)
  		snake_case__  					=      torch.tensor([[0, 0, 0, 1, 1]]		)
  		with torch.no_grad():
  			snake_case__  					=      model(UpperCamelCase_			,					token_type_ids=UpperCamelCase_			,					output_hidden_states=UpperCamelCase_		)
  
  		snake_case__  					=      1152
  
  		snake_case__  					=      torch.Size((1, 5, hidden_size)		)
  		self.assertEqual(output['last_hidden_state'].shape			,					UpperCamelCase_		)
  
  		snake_case__  					=      torch.tensor(
  		    [
  		        [
  		            [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
  		            [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
  		            [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
  		            [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
  		            [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
  		        ]
  		    ]		)
  
  		# Running on the original tf implementation gives slightly different results here.
  		# Not clear why this variations is present
  		# TODO: Find reason for discrepancy
  		# expected_original_implementation = [[
  		#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
  		#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
  		#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
  		#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
  		#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
  		# ]]
  
  		self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]			,					UpperCamelCase_			,					atol=1E-4		)		)
  
  
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  def 	__lowerCamelCase (  UpperCAmelCase_       )      ->int:
  	return 1 if input_a == 0 else 0
  
  
  
  
  def 	__lowerCamelCase (  )      ->None:
  	assert not_gate(0       ) == 1
  	assert not_gate(1       ) == 0
  
  
  if __name__ == "__main__":
  			print(not_gate(0))
  			print(not_gate(1))
  
  
  
  