code: 
  
  
  
  """simple docstring"""
  
  
  
  
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
   import torch
  
   from transformers import (
       RemBertConfig,
       RemBertForCausalLM,
       RemBertForMaskedLM,
       RemBertForMultipleChoice,
       RemBertForQuestionAnswering,
       RemBertForSequenceClassification,
       RemBertForTokenClassification,
       RemBertModel,
   )
   from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class 			UpperCAmelCase_						:
  
  
  
        def __init__(  self  :			int  ,			A  :			List[Any]  ,			A  :			Tuple=1_3  ,			A  :			Tuple=7  ,			A  :			int=True  ,			A  :			Dict=True  ,			A  :			List[Any]=True  ,			A  :			str=True  ,			A  :			Dict=9_9  ,			A  :			str=3_2  ,			A  :			Union[str, Any]=1_8  ,			A  :			Any=4_3  ,			A  :			Optional[int]=5  ,			A  :			Tuple=4  ,			A  :			List[str]=3_7  ,			A  :			Tuple="gelu"  ,			A  :			Optional[Any]=0.1  ,			A  :			int=0.1  ,			A  :			str=5_1_2  ,			A  :			Optional[int]=1_6  ,			A  :			List[Any]=2  ,			A  :			int=0.02  ,			A  :			str=3  ,			A  :			str=4  ,			A  :			Optional[int]=None  ,			):
             _UpperCAmelCase    :			Union[str, Any]              =					parent
             _UpperCAmelCase    :			Any              =					batch_size
             _UpperCAmelCase    :			Optional[Any]              =					seq_length
             _UpperCAmelCase    :			str              =					is_training
             _UpperCAmelCase    :			List[str]              =					use_input_mask
             _UpperCAmelCase    :			Any              =					use_token_type_ids
             _UpperCAmelCase    :			Union[str, Any]              =					use_labels
             _UpperCAmelCase    :			Union[str, Any]              =					vocab_size
             _UpperCAmelCase    :			Tuple              =					hidden_size
             _UpperCAmelCase    :			Optional[Any]              =					input_embedding_size
             _UpperCAmelCase    :			Optional[Any]              =					output_embedding_size
             _UpperCAmelCase    :			Tuple              =					num_hidden_layers
             _UpperCAmelCase    :			List[str]              =					num_attention_heads
             _UpperCAmelCase    :			Union[str, Any]              =					intermediate_size
             _UpperCAmelCase    :			Union[str, Any]              =					hidden_act
             _UpperCAmelCase    :			Union[str, Any]              =					hidden_dropout_prob
             _UpperCAmelCase    :			Dict              =					attention_probs_dropout_prob
             _UpperCAmelCase    :			Optional[int]              =					max_position_embeddings
             _UpperCAmelCase    :			Dict              =					type_vocab_size
             _UpperCAmelCase    :			List[Any]              =					type_sequence_label_size
             _UpperCAmelCase    :			Optional[int]              =					initializer_range
             _UpperCAmelCase    :			Optional[Any]              =					num_labels
             _UpperCAmelCase    :			int              =					num_choices
             _UpperCAmelCase    :			Dict              =					scope
  
  
  
        def 		snake_case_      (  self  :			List[Any]		):
             _UpperCAmelCase    :			int              =					ids_tensor([self.batch_size, self.seq_length]  ,			self.vocab_size		)
  
             _UpperCAmelCase    :			List[str]              =					None
             if self.use_input_mask:
                  _UpperCAmelCase    :			List[Any]              =					random_attention_mask([self.batch_size, self.seq_length]		)
  
             _UpperCAmelCase    :			List[Any]              =					None
             if self.use_token_type_ids:
                  _UpperCAmelCase    :			int              =					ids_tensor([self.batch_size, self.seq_length]  ,			self.type_vocab_size		)
  
             _UpperCAmelCase    :			List[Any]              =					None
             _UpperCAmelCase    :			List[str]              =					None
             _UpperCAmelCase    :			Dict              =					None
             if self.use_labels:
                  _UpperCAmelCase    :			List[str]              =					ids_tensor([self.batch_size]  ,			self.type_sequence_label_size		)
                  _UpperCAmelCase    :			Union[str, Any]              =					ids_tensor([self.batch_size, self.seq_length]  ,			self.num_labels		)
                  _UpperCAmelCase    :			Dict              =					ids_tensor([self.batch_size]  ,			self.num_choices		)
  
             _UpperCAmelCase    :			Optional[Any]              =					RemBertConfig(
                 vocab_size=self.vocab_size  ,			hidden_size=self.hidden_size  ,			input_embedding_size=self.input_embedding_size  ,			output_embedding_size=self.output_embedding_size  ,			num_hidden_layers=self.num_hidden_layers  ,			num_attention_heads=self.num_attention_heads  ,			intermediate_size=self.intermediate_size  ,			hidden_act=self.hidden_act  ,			hidden_dropout_prob=self.hidden_dropout_prob  ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,			max_position_embeddings=self.max_position_embeddings  ,			type_vocab_size=self.type_vocab_size  ,			is_decoder=UpperCAmelCase_  ,			initializer_range=self.initializer_range  ,			)
  
             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
        def 		snake_case_      (  self  :			str		):
             (
                 _UpperCAmelCase  
             )    :			Optional[Any]              =					self.prepare_config_and_inputs()
  
             _UpperCAmelCase    :			Tuple              =					True
             _UpperCAmelCase    :			Optional[Any]              =					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
             _UpperCAmelCase    :			str              =					ids_tensor([self.batch_size, self.seq_length]  ,			vocab_size=2		)
  
             return (
                 config,
                 input_ids,
                 token_type_ids,
                 input_mask,
                 sequence_labels,
                 token_labels,
                 choice_labels,
                 encoder_hidden_states,
                 encoder_attention_mask,
             )
  
  
  
        def 		snake_case_      (  self  :			Tuple  ,			A  :			Any  ,			A  :			Tuple  ,			A  :			Tuple  ,			A  :			str  ,			A  :			Dict  ,			A  :			List[Any]  ,			A  :			int		):
             _UpperCAmelCase    :			str              =					RemBertModel(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			Optional[int]              =					model(UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_		)
             _UpperCAmelCase    :			str              =					model(UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_		)
             _UpperCAmelCase    :			Optional[Any]              =					model(UpperCAmelCase_		)
             self.parent.assertEqual(result.last_hidden_state.shape  ,			(self.batch_size, self.seq_length, self.hidden_size)		)
  
  
  
        def 		snake_case_      (  self  :			str  ,			A  :			Any  ,			A  :			List[Any]  ,			A  :			Tuple  ,			A  :			int  ,			A  :			Union[str, Any]  ,			A  :			Any  ,			A  :			Any  ,			A  :			List[Any]  ,			A  :			Union[str, Any]  ,			):
             _UpperCAmelCase    :			List[Any]              =					True
             _UpperCAmelCase    :			Optional[int]              =					RemBertModel(UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			int              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			encoder_hidden_states=UpperCAmelCase_  ,			encoder_attention_mask=UpperCAmelCase_  ,			)
             _UpperCAmelCase    :			Optional[int]              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			encoder_hidden_states=UpperCAmelCase_  ,			)
             _UpperCAmelCase    :			Optional[Any]              =					model(UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_		)
             self.parent.assertEqual(result.last_hidden_state.shape  ,			(self.batch_size, self.seq_length, self.hidden_size)		)
  
  
  
        def 		snake_case_      (  self  :			Tuple  ,			A  :			Dict  ,			A  :			int  ,			A  :			str  ,			A  :			Union[str, Any]  ,			A  :			Optional[int]  ,			A  :			Dict  ,			A  :			List[Any]  ,			A  :			Any  ,			A  :			Any  ,			):
             _UpperCAmelCase    :			Any              =					RemBertForCausalLM(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			List[str]              =					model(UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			labels=UpperCAmelCase_		)
             self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.seq_length, self.vocab_size)		)
  
  
  
        def 		snake_case_      (  self  :			Union[str, Any]  ,			A  :			List[str]  ,			A  :			Any  ,			A  :			Any  ,			A  :			Dict  ,			A  :			Optional[Any]  ,			A  :			str  ,			A  :			Union[str, Any]		):
             _UpperCAmelCase    :			Any              =					RemBertForMaskedLM(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			List[str]              =					model(UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			labels=UpperCAmelCase_		)
             self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.seq_length, self.vocab_size)		)
  
  
  
        def 		snake_case_      (  self  :			Tuple  ,			A  :			Optional[int]  ,			A  :			Optional[int]  ,			A  :			List[str]  ,			A  :			List[Any]  ,			A  :			Tuple  ,			A  :			Any  ,			A  :			List[Any]  ,			A  :			Union[str, Any]  ,			A  :			Tuple  ,			):
             _UpperCAmelCase    :			Optional[Any]              =					True
             _UpperCAmelCase    :			Dict              =					True
             _UpperCAmelCase    :			List[Any]              =					RemBertForCausalLM(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
  
             # first forward pass
             _UpperCAmelCase    :			int              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			encoder_hidden_states=UpperCAmelCase_  ,			encoder_attention_mask=UpperCAmelCase_  ,			use_cache=UpperCAmelCase_  ,			)
             _UpperCAmelCase    :			Optional[Any]              =					outputs.past_key_values
  
             # create hypothetical multiple next token and extent to next_input_ids
             _UpperCAmelCase    :			List[Any]              =					ids_tensor((self.batch_size, 3)  ,			config.vocab_size		)
             _UpperCAmelCase    :			Tuple              =					ids_tensor((self.batch_size, 3)  ,			vocab_size=2		)
  
             # append to next input_ids and
             _UpperCAmelCase    :			int              =					torch.cat([input_ids, next_tokens]  ,			dim=-1		)
             _UpperCAmelCase    :			Any              =					torch.cat([input_mask, next_mask]  ,			dim=-1		)
  
             _UpperCAmelCase    :			List[Any]              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			encoder_hidden_states=UpperCAmelCase_  ,			encoder_attention_mask=UpperCAmelCase_  ,			output_hidden_states=UpperCAmelCase_  ,			)['hidden_states'][0]
             _UpperCAmelCase    :			Any              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			encoder_hidden_states=UpperCAmelCase_  ,			encoder_attention_mask=UpperCAmelCase_  ,			past_key_values=UpperCAmelCase_  ,			output_hidden_states=UpperCAmelCase_  ,			)['hidden_states'][0]
  
             # select random slice
             _UpperCAmelCase    :			List[Any]              =					ids_tensor((1,)  ,			output_from_past.shape[-1]		).item()
             _UpperCAmelCase    :			Tuple              =					output_from_no_past[:, -3:, random_slice_idx].detach()
             _UpperCAmelCase    :			Optional[Any]              =					output_from_past[:, :, random_slice_idx].detach()
  
             self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)
  
             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(UpperCAmelCase_  ,			UpperCAmelCase_  ,			atol=1e-3		)		)
  
  
  
        def 		snake_case_      (  self  :			Optional[int]  ,			A  :			List[Any]  ,			A  :			Tuple  ,			A  :			Any  ,			A  :			Dict  ,			A  :			Dict  ,			A  :			Dict  ,			A  :			Optional[Any]		):
             _UpperCAmelCase    :			List[str]              =					RemBertForQuestionAnswering(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			Tuple              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			start_positions=UpperCAmelCase_  ,			end_positions=UpperCAmelCase_  ,			)
             self.parent.assertEqual(result.start_logits.shape  ,			(self.batch_size, self.seq_length)		)
             self.parent.assertEqual(result.end_logits.shape  ,			(self.batch_size, self.seq_length)		)
  
  
  
        def 		snake_case_      (  self  :			int  ,			A  :			Dict  ,			A  :			List[str]  ,			A  :			Union[str, Any]  ,			A  :			Union[str, Any]  ,			A  :			Optional[Any]  ,			A  :			List[str]  ,			A  :			Any		):
             _UpperCAmelCase    :			Any              =					self.num_labels
             _UpperCAmelCase    :			Tuple              =					RemBertForSequenceClassification(UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			Optional[int]              =					model(UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			labels=UpperCAmelCase_		)
             self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.num_labels)		)
  
  
  
        def 		snake_case_      (  self  :			List[str]  ,			A  :			List[str]  ,			A  :			Optional[Any]  ,			A  :			int  ,			A  :			Any  ,			A  :			List[str]  ,			A  :			Tuple  ,			A  :			List[Any]		):
             _UpperCAmelCase    :			List[str]              =					self.num_labels
             _UpperCAmelCase    :			List[Any]              =					RemBertForTokenClassification(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			Union[str, Any]              =					model(UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			labels=UpperCAmelCase_		)
             self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.seq_length, self.num_labels)		)
  
  
  
        def 		snake_case_      (  self  :			List[str]  ,			A  :			List[str]  ,			A  :			Any  ,			A  :			Union[str, Any]  ,			A  :			List[str]  ,			A  :			int  ,			A  :			List[Any]  ,			A  :			Union[str, Any]		):
             _UpperCAmelCase    :			Optional[Any]              =					self.num_choices
             _UpperCAmelCase    :			int              =					RemBertForMultipleChoice(config=UpperCAmelCase_		)
             model.to(UpperCAmelCase_		)
             model.eval()
             _UpperCAmelCase    :			Tuple              =					input_ids.unsqueeze(1		).expand(-1  ,			self.num_choices  ,			-1		).contiguous()
             _UpperCAmelCase    :			Tuple              =					token_type_ids.unsqueeze(1		).expand(-1  ,			self.num_choices  ,			-1		).contiguous()
             _UpperCAmelCase    :			List[Any]              =					input_mask.unsqueeze(1		).expand(-1  ,			self.num_choices  ,			-1		).contiguous()
             _UpperCAmelCase    :			List[str]              =					model(
                 UpperCAmelCase_  ,			attention_mask=UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			labels=UpperCAmelCase_  ,			)
             self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.num_choices)		)
  
  
  
        def 		snake_case_      (  self  :			Tuple		):
             _UpperCAmelCase    :			Optional[int]              =					self.prepare_config_and_inputs()
             (
                 _UpperCAmelCase  
             )    :			List[Any]              =					config_and_inputs
             _UpperCAmelCase    :			str              =					{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
             return config, inputs_dict
  
  
  
  
  
  
  
  @require_torch
  class 			UpperCAmelCase_						(       _UpperCamelCase ,    _UpperCamelCase ,    unittest.TestCase		):
        __SCREAMING_SNAKE_CASE		:     str				  =				(
            (
                RemBertModel,
                RemBertForMaskedLM,
                RemBertForCausalLM,
                RemBertForMultipleChoice,
                RemBertForQuestionAnswering,
                RemBertForSequenceClassification,
                RemBertForTokenClassification,
            )
            if is_torch_available()
            else ()
        )
        __SCREAMING_SNAKE_CASE		:     Any				  =				(RemBertForCausalLM,) if is_torch_available() else ()
        __SCREAMING_SNAKE_CASE		:     List[Any]				  =				(
            {
                "feature-extraction": RemBertModel,
                "fill-mask": RemBertForMaskedLM,
                "question-answering": RemBertForQuestionAnswering,
                "text-classification": RemBertForSequenceClassification,
                "text-generation": RemBertForCausalLM,
                "token-classification": RemBertForTokenClassification,
                "zero-shot": RemBertForSequenceClassification,
            }
            if is_torch_available()
            else {}
        )
  
  
  
        def 		snake_case_      (  self  :			Optional[int]		):
             _UpperCAmelCase    :			str              =					RemBertModelTester(self		)
             _UpperCAmelCase    :			List[str]              =					ConfigTester(self  ,			config_class=UpperCAmelCase_  ,			hidden_size=3_7		)
  
  
  
        def 		snake_case_      (  self  :			Tuple		):
             self.config_tester.run_common_tests()
  
  
  
        def 		snake_case_      (  self  :			Any		):
             _UpperCAmelCase    :			Union[str, Any]              =					self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			List[Any]		):
             _UpperCAmelCase    :			List[Any]              =					self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                  _UpperCAmelCase    :			str              =					type
                  self.model_tester.create_and_check_model(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			str		):
             _UpperCAmelCase    :			Tuple              =					self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			Tuple		):
             _UpperCAmelCase    :			Tuple              =					self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			Tuple		):
             _UpperCAmelCase    :			Dict              =					self.model_tester.prepare_config_and_inputs_for_decoder()
             self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			List[str]		):
             _UpperCAmelCase    :			Dict              =					self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			Any		):
             _UpperCAmelCase    :			List[Any]              =					self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			Dict		):
             _UpperCAmelCase    :			Any              =					self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			Optional[Any]		):
             _UpperCAmelCase    :			Optional[int]              =					self.model_tester.prepare_config_and_inputs_for_decoder()
             self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase_		)
  
  
  
        def 		snake_case_      (  self  :			Optional[int]		):
             # This regression test was failing with PyTorch < 1.3
             (
                 _UpperCAmelCase  
             )    :			Union[str, Any]              =					self.model_tester.prepare_config_and_inputs_for_decoder()
  
             _UpperCAmelCase    :			List[str]              =					None
  
             self.model_tester.create_and_check_model_as_decoder(
                 UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			)
  
  
  
        @slow
        def 		snake_case_      (  self  :			Tuple		):
             for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  _UpperCAmelCase    :			int              =					RemBertModel.from_pretrained(UpperCAmelCase_		)
                  self.assertIsNotNone(UpperCAmelCase_		)
  
  
  
  
  
  
  
  @require_torch
  class 			UpperCAmelCase_						(       unittest.TestCase		):
  
  
  
        @slow
        def 		snake_case_      (  self  :			List[Any]		):
             # Test exact values at the last hidden layer
             _UpperCAmelCase    :			Union[str, Any]              =					RemBertModel.from_pretrained("google/rembert"		)
             _UpperCAmelCase    :			str              =					torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]		)
             _UpperCAmelCase    :			List[str]              =					torch.tensor([[0, 0, 0, 1, 1]]		)
             with torch.no_grad():
                  _UpperCAmelCase    :			str              =					model(UpperCAmelCase_  ,			token_type_ids=UpperCAmelCase_  ,			output_hidden_states=UpperCAmelCase_		)
  
             _UpperCAmelCase    :			int              =					1_1_5_2
  
             _UpperCAmelCase    :			Union[str, Any]              =					torch.Size((1, 5, hidden_size)		)
             self.assertEqual(output["last_hidden_state"].shape  ,			UpperCAmelCase_		)
  
             _UpperCAmelCase    :			Dict              =					torch.tensor(
                 [
                     [
                         [0.0_754, -0.2_022, 0.1_904],
                         [-0.3_354, -0.3_692, -0.4_791],
                         [-0.2_314, -0.6_729, -0.0_749],
                         [-0.0_396, -0.3_105, -0.4_234],
                         [-0.1_571, -0.0_525, 0.5_353],
                     ]
                 ]		)
  
             # Running on the original tf implementation gives slightly different results here.
             # Not clear why this variations is present
             # TODO: Find reason for discrepancy
             # expected_original_implementation = [[
             #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
             #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
             #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
             #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
             #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
             # ]]
  
             self.assertTrue(torch.allclose(output["last_hidden_state"][:, :, :3]  ,			UpperCAmelCase_  ,			atol=1e-4		)		)
  
style_context: 
  
  
  
  """simple docstring"""
  
  
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
   from .tokenization_xglm import XGLMTokenizer
  else:
   _lowerCAmelCase       :      Tuple							=							None
  
  
  _lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)
  
  _lowerCAmelCase       :      str							=							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}
  
  _lowerCAmelCase       :      Optional[Any]							=							{
      "vocab_file": {
          "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
      },
      "tokenizer_file": {
          "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
      },
  }
  
  _lowerCAmelCase       :      Union[str, Any]							=							{
      "facebook/xglm-564M": 20_48,
  }
  
  
  
  
  
  
  class 			UpperCAmelCase_						(       _UpperCamelCase		):
        __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
        __SCREAMING_SNAKE_CASE		:     List[str]				  =				PRETRAINED_VOCAB_FILES_MAP
        __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __SCREAMING_SNAKE_CASE		:     int				  =				['input_ids', 'attention_mask']
        __SCREAMING_SNAKE_CASE		:     str				  =				XGLMTokenizer
  
  
  
        def __init__(  self  :			List[Any]  ,			A  :			Union[str, Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Optional[Any]="<s>"  ,			A  :			Optional[int]="</s>"  ,			A  :			str="</s>"  ,			A  :			int="<s>"  ,			A  :			int="<unk>"  ,			A  :			List[str]="<pad>"  ,			**A  :			Tuple  ,			):
             # Compatibility with the original tokenizer
             _UpperCAmelCase    :			Union[str, Any]              =					7
             _UpperCAmelCase    :			Optional[int]              =					[f'<madeupword{i}>' for i in range(self.num_madeup_words		)]
  
             _UpperCAmelCase    :			Any              =					kwargs.get("additional_special_tokens"  ,			[]		)
             kwargs["additional_special_tokens"] += [
                 word for word in madeup_words if word not in kwargs["additional_special_tokens"]
             ]
  
             super().__init__(
                 A  ,			tokenizer_file=A  ,			bos_token=A  ,			eos_token=A  ,			sep_token=A  ,			cls_token=A  ,			unk_token=A  ,			pad_token=A  ,			**A  ,			)
  
             _UpperCAmelCase    :			Any              =					vocab_file
             _UpperCAmelCase    :			Union[str, Any]              =					False if not self.vocab_file else True
  
  
  
        def 		snake_case_      (  self  :			List[str]  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
  
             if token_ids_a is None:
                  return [self.sep_token_id] + token_ids_a
             _UpperCAmelCase    :			Tuple              =					[self.sep_token_id]
             return sep + token_ids_a + sep + sep + token_ids_a
  
  
  
        def 		snake_case_      (  self  :			str  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
             _UpperCAmelCase    :			Optional[int]              =					[self.sep_token_id]
  
             if token_ids_a is None:
                  return len(sep + token_ids_a		) * [0]
             return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]
  
  
  
        def 		snake_case_      (  self  :			Dict  ,			A  :			str  ,			A  :			Optional[str] = None		):
             if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                      "tokenizer."		)
  
             if not os.path.isdir(A		):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory.'		)
                  return
             _UpperCAmelCase    :			str              =					os.path.join(
                 A  ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
  
             if os.path.abspath(self.vocab_file		) != os.path.abspath(A		):
                  copyfile(self.vocab_file  ,			A		)
  
             return (out_vocab_file,)
  