



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    RemBertConfig,
		    RemBertForCausalLM,
		    RemBertForMaskedLM,
		    RemBertForMultipleChoice,
		    RemBertForQuestionAnswering,
		    RemBertForSequenceClassification,
		    RemBertForTokenClassification,
		    RemBertModel,
		)
		from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class      snake_case__       :







						"""simple docstring"""




						def __init__(     self      ,     __lowercase      ,     __lowercase=1_3      ,     __lowercase=7      ,     __lowercase=True      ,     __lowercase=True      ,     __lowercase=True      ,     __lowercase=True      ,     __lowercase=9_9      ,     __lowercase=3_2      ,     __lowercase=1_8      ,     __lowercase=4_3      ,     __lowercase=5      ,     __lowercase=4      ,     __lowercase=3_7      ,     __lowercase="gelu"      ,     __lowercase=0.1      ,     __lowercase=0.1      ,     __lowercase=5_1_2      ,     __lowercase=1_6      ,     __lowercase=2      ,     __lowercase=0.0_2      ,     __lowercase=3      ,     __lowercase=4      ,     __lowercase=None      ,     )      ->   List[Any]:
									"""simple docstring"""




									a__     :		List[str]		 =						parent
									a__     :		Any		 =						batch_size
									a__     :		Any		 =						seq_length
									a__     :		Optional[int]		 =						is_training
									a__     :		List[Any]		 =						use_input_mask
									a__     :		Optional[int]		 =						use_token_type_ids
									a__     :		Optional[Any]		 =						use_labels
									a__     :		Any		 =						vocab_size
									a__     :		Tuple		 =						hidden_size
									a__     :		List[Any]		 =						input_embedding_size
									a__     :		Dict		 =						output_embedding_size
									a__     :		Optional[Any]		 =						num_hidden_layers
									a__     :		Any		 =						num_attention_heads
									a__     :		List[Any]		 =						intermediate_size
									a__     :		Optional[int]		 =						hidden_act
									a__     :		Any		 =						hidden_dropout_prob
									a__     :		List[str]		 =						attention_probs_dropout_prob
									a__     :		Any		 =						max_position_embeddings
									a__     :		Optional[Any]		 =						type_vocab_size
									a__     :		int		 =						type_sequence_label_size
									a__     :		Union[str, Any]		 =						initializer_range
									a__     :		Optional[int]		 =						num_labels
									a__     :		Any		 =						num_choices
									a__     :		Optional[Any]		 =						scope

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Union[str, Any]:
									"""simple docstring"""




									a__     :		int		 =						ids_tensor([self.batch_size, self.seq_length]      ,     self.vocab_size						)

									a__     :		Tuple		 =						None
									if self.use_input_mask:
												a__     :		Optional[Any]		 =						random_attention_mask([self.batch_size, self.seq_length]						)

									a__     :		Any		 =						None
									if self.use_token_type_ids:
												a__     :		Optional[int]		 =						ids_tensor([self.batch_size, self.seq_length]      ,     self.type_vocab_size						)

									a__     :		Dict		 =						None
									a__     :		Optional[Any]		 =						None
									a__     :		Dict		 =						None
									if self.use_labels:
												a__     :		Tuple		 =						ids_tensor([self.batch_size]      ,     self.type_sequence_label_size						)
												a__     :		Optional[Any]		 =						ids_tensor([self.batch_size, self.seq_length]      ,     self.num_labels						)
												a__     :		Tuple		 =						ids_tensor([self.batch_size]      ,     self.num_choices						)

									a__     :		Any		 =						RemBertConfig(
									    vocab_size=self.vocab_size      ,     hidden_size=self.hidden_size      ,     input_embedding_size=self.input_embedding_size      ,     output_embedding_size=self.output_embedding_size      ,     num_hidden_layers=self.num_hidden_layers      ,     num_attention_heads=self.num_attention_heads      ,     intermediate_size=self.intermediate_size      ,     hidden_act=self.hidden_act      ,     hidden_dropout_prob=self.hidden_dropout_prob      ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,     max_position_embeddings=self.max_position_embeddings      ,     type_vocab_size=self.type_vocab_size      ,     is_decoder=__lowercase      ,     initializer_range=self.initializer_range      ,     )

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Dict:
									"""simple docstring"""




									(
									    (
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		
									)     :		Tuple		 =						self.prepare_config_and_inputs()

									a__     :		Optional[int]		 =						True
									a__     :		Tuple		 =						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
									a__     :		Any		 =						ids_tensor([self.batch_size, self.seq_length]      ,     vocab_size=2						)

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    encoder_hidden_states,
									    encoder_attention_mask,
									)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase						)      ->   int:
									"""simple docstring"""




									a__     :		str		 =						RemBertModel(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		int		 =						model(__lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase						)
									a__     :		Dict		 =						model(__lowercase      ,     token_type_ids=__lowercase						)
									a__     :		List[str]		 =						model(__lowercase						)
									self.parent.assertEqual(result.last_hidden_state.shape      ,     (self.batch_size, self.seq_length, self.hidden_size)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     )      ->   Optional[int]:
									"""simple docstring"""




									a__     :		Optional[int]		 =						True
									a__     :		Tuple		 =						RemBertModel(__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		Optional[Any]		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     encoder_hidden_states=__lowercase      ,     encoder_attention_mask=__lowercase      ,     )
									a__     :		Optional[int]		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     encoder_hidden_states=__lowercase      ,     )
									a__     :		str		 =						model(__lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase						)
									self.parent.assertEqual(result.last_hidden_state.shape      ,     (self.batch_size, self.seq_length, self.hidden_size)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     )      ->   Dict:
									"""simple docstring"""




									a__     :		List[str]		 =						RemBertForCausalLM(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		Any		 =						model(__lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     labels=__lowercase						)
									self.parent.assertEqual(result.logits.shape      ,     (self.batch_size, self.seq_length, self.vocab_size)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase						)      ->   List[str]:
									"""simple docstring"""




									a__     :		str		 =						RemBertForMaskedLM(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		List[Any]		 =						model(__lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     labels=__lowercase						)
									self.parent.assertEqual(result.logits.shape      ,     (self.batch_size, self.seq_length, self.vocab_size)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     )      ->   List[str]:
									"""simple docstring"""




									a__     :		Optional[int]		 =						True
									a__     :		Union[str, Any]		 =						True
									a__     :		Optional[Any]		 =						RemBertForCausalLM(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()

									# first forward pass
									a__     :		Union[str, Any]		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     encoder_hidden_states=__lowercase      ,     encoder_attention_mask=__lowercase      ,     use_cache=__lowercase      ,     )
									a__     :		Dict		 =						outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									a__     :		Tuple		 =						ids_tensor((self.batch_size, 3)      ,     config.vocab_size						)
									a__     :		int		 =						ids_tensor((self.batch_size, 3)      ,     vocab_size=2						)

									# append to next input_ids and
									a__     :		Optional[int]		 =						torch.cat([input_ids, next_tokens]      ,     dim=-1						)
									a__     :		str		 =						torch.cat([input_mask, next_mask]      ,     dim=-1						)

									a__     :		Optional[int]		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     encoder_hidden_states=__lowercase      ,     encoder_attention_mask=__lowercase      ,     output_hidden_states=__lowercase      ,     )["""hidden_states"""][0]
									a__     :		List[Any]		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     encoder_hidden_states=__lowercase      ,     encoder_attention_mask=__lowercase      ,     past_key_values=__lowercase      ,     output_hidden_states=__lowercase      ,     )["""hidden_states"""][0]

									# select random slice
									a__     :		int		 =						ids_tensor((1,)      ,     output_from_past.shape[-1]						).item()
									a__     :		Union[str, Any]		 =						output_from_no_past[:, -3:, random_slice_idx].detach()
									a__     :		Union[str, Any]		 =						output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(__lowercase      ,     __lowercase      ,     atol=1E-3						)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase						)      ->   int:
									"""simple docstring"""




									a__     :		List[str]		 =						RemBertForQuestionAnswering(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		Any		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     start_positions=__lowercase      ,     end_positions=__lowercase      ,     )
									self.parent.assertEqual(result.start_logits.shape      ,     (self.batch_size, self.seq_length)						)
									self.parent.assertEqual(result.end_logits.shape      ,     (self.batch_size, self.seq_length)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		List[str]		 =						self.num_labels
									a__     :		Optional[int]		 =						RemBertForSequenceClassification(__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		Any		 =						model(__lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     labels=__lowercase						)
									self.parent.assertEqual(result.logits.shape      ,     (self.batch_size, self.num_labels)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		Any		 =						self.num_labels
									a__     :		Dict		 =						RemBertForTokenClassification(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		Any		 =						model(__lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     labels=__lowercase						)
									self.parent.assertEqual(result.logits.shape      ,     (self.batch_size, self.seq_length, self.num_labels)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase						)      ->   Union[str, Any]:
									"""simple docstring"""




									a__     :		Union[str, Any]		 =						self.num_choices
									a__     :		Optional[Any]		 =						RemBertForMultipleChoice(config=__lowercase						)
									model.to(__lowercase						)
									model.eval()
									a__     :		Optional[Any]		 =						input_ids.unsqueeze(1						).expand(-1      ,     self.num_choices      ,     -1						).contiguous()
									a__     :		List[Any]		 =						token_type_ids.unsqueeze(1						).expand(-1      ,     self.num_choices      ,     -1						).contiguous()
									a__     :		List[str]		 =						input_mask.unsqueeze(1						).expand(-1      ,     self.num_choices      ,     -1						).contiguous()
									a__     :		Dict		 =						model(
									    __lowercase      ,     attention_mask=__lowercase      ,     token_type_ids=__lowercase      ,     labels=__lowercase      ,     )
									self.parent.assertEqual(result.logits.shape      ,     (self.batch_size, self.num_choices)						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Tuple:
									"""simple docstring"""




									a__     :		List[Any]		 =						self.prepare_config_and_inputs()
									(
									    (
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		
									)     :		Any		 =						config_and_inputs
									a__     :		List[Any]		 =						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
									return config, inputs_dict




@require_torch
class      snake_case__       (A__      ,			A__      ,			unittest.TestCase	):







						"""simple docstring"""




						__lowerCAmelCase  :List[Any]			      =  (
						    (
						        RemBertModel,
						        RemBertForMaskedLM,
						        RemBertForCausalLM,
						        RemBertForMultipleChoice,
						        RemBertForQuestionAnswering,
						        RemBertForSequenceClassification,
						        RemBertForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						__lowerCAmelCase  :Tuple			      =  (RemBertForCausalLM,) if is_torch_available() else ()
						__lowerCAmelCase  :Any			      =  (
						    {
						        "feature-extraction": RemBertModel,
						        "fill-mask": RemBertForMaskedLM,
						        "question-answering": RemBertForQuestionAnswering,
						        "text-classification": RemBertForSequenceClassification,
						        "text-generation": RemBertForCausalLM,
						        "token-classification": RemBertForTokenClassification,
						        "zero-shot": RemBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		Optional[int]		 =						RemBertModelTester(self						)
									a__     :		Tuple		 =						ConfigTester(self      ,     config_class=__lowercase      ,     hidden_size=3_7						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Any:
									"""simple docstring"""




									self.config_tester.run_common_tests()

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Any:
									"""simple docstring"""




									a__     :		Optional[int]		 =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		Any		 =						self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
												a__     :		int		 =						type
												self.model_tester.create_and_check_model(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		int		 =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[Any]:
									"""simple docstring"""




									a__     :		List[str]		 =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Union[str, Any]:
									"""simple docstring"""




									a__     :		Union[str, Any]		 =						self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_decoder_model_past_large_inputs(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   str:
									"""simple docstring"""




									a__     :		Optional[int]		 =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Any:
									"""simple docstring"""




									a__     :		List[Any]		 =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[Any]:
									"""simple docstring"""




									a__     :		Optional[int]		 =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		Union[str, Any]		 =						self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_model_as_decoder(*__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									(
									    (
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		(
									    a__
									) ,		
									)     :		Tuple		 =						self.model_tester.prepare_config_and_inputs_for_decoder()

									a__     :		List[str]		 =						None

									self.model_tester.create_and_check_model_as_decoder(
									    __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     __lowercase      ,     )

						@slow
						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   List[Any]:
									"""simple docstring"""




									for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												a__     :		List[str]		 =						RemBertModel.from_pretrained(__lowercase						)
												self.assertIsNotNone(__lowercase						)




@require_torch
class      snake_case__       (unittest.TestCase	):







						"""simple docstring"""




						@slow
						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		str		 =						RemBertModel.from_pretrained("""google/rembert"""						)
									a__     :		Optional[int]		 =						torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]						)
									a__     :		Optional[int]		 =						torch.tensor([[0, 0, 0, 1, 1]]						)
									with torch.no_grad():
												a__     :		Union[str, Any]		 =						model(__lowercase      ,     token_type_ids=__lowercase      ,     output_hidden_states=__lowercase						)

									a__     :		Tuple		 =						1_1_5_2

									a__     :		Any		 =						torch.Size((1, 5, hidden_size)						)
									self.assertEqual(output["""last_hidden_state"""].shape      ,     __lowercase						)

									a__     :		Union[str, Any]		 =						torch.tensor(
									    [
									        [
									            [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
									            [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
									            [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
									            [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
									            [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
									        ]
									    ]						)

									# Running on the original tf implementation gives slightly different results here.
									# Not clear why this variations is present
									# TODO: Find reason for discrepancy
									# expected_original_implementation = [[
									#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
									#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
									#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
									#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
									#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
									# ]]

									self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3]      ,     __lowercase      ,     atol=1E-4						)						)









import decimal

def 			lowerCAmelCase_  (						)							-> int:







			"""simple docstring"""
			a__     :		Optional[Any]		 =						0
			a__     :		Optional[int]		 =						decimal.Context(prec=105)
			for i in range(2		,     100):
						a__     :		int		 =						decimal.Decimal(_lowercase)
						a__     :		Optional[int]		 =						number.sqrt(_lowercase)
						if len(str(_lowercase)) > 1:
									answer += int(str(_lowercase)[0])
									a__     :		Tuple		 =						str(_lowercase)[2:101]
									answer += sum(int(_lowercase) for x in sqrt_number_str)
			return answer


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		print(f'{solution() = }')




