

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           RemBertConfig,
           RemBertForCausalLM,
           RemBertForMaskedLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
           RemBertModel,
       )
       from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 		__magic_name__    :

   '''simple docstring'''




   def __init__(       self,					lowercase_,					lowercase_=13,					lowercase_=7,					lowercase_=True,					lowercase_=True,					lowercase_=True,					lowercase_=True,					lowercase_=99,					lowercase_=32,					lowercase_=18,					lowercase_=43,					lowercase_=5,					lowercase_=4,					lowercase_=37,					lowercase_="gelu",					lowercase_=0.1,					lowercase_=0.1,					lowercase_=512,					lowercase_=16,					lowercase_=2,					lowercase_=0.02,					lowercase_=3,					lowercase_=4,					lowercase_=None,					)							->					str:






     """simple docstring"""






     a__						=parent
     a__						=batch_size
     a__						=seq_length
     a__						=is_training
     a__						=use_input_mask
     a__						=use_token_type_ids
     a__						=use_labels
     a__						=vocab_size
     a__						=hidden_size
     a__						=input_embedding_size
     a__						=output_embedding_size
     a__						=num_hidden_layers
     a__						=num_attention_heads
     a__						=intermediate_size
     a__						=hidden_act
     a__						=hidden_dropout_prob
     a__						=attention_probs_dropout_prob
     a__						=max_position_embeddings
     a__						=type_vocab_size
     a__						=type_sequence_label_size
     a__						=initializer_range
     a__						=num_labels
     a__						=num_choices
     a__						=scope



   def 				_UpperCAmelCase				(       self )							->					Union[str, Any]:






     """simple docstring"""






     a__						=ids_tensor([self.batch_size, self.seq_length],					self.vocab_size )

     a__						=None
     if self.use_input_mask:
       a__						=random_attention_mask([self.batch_size, self.seq_length] )

     a__						=None
     if self.use_token_type_ids:
       a__						=ids_tensor([self.batch_size, self.seq_length],					self.type_vocab_size )

     a__						=None
     a__						=None
     a__						=None
     if self.use_labels:
       a__						=ids_tensor([self.batch_size],					self.type_sequence_label_size )
       a__						=ids_tensor([self.batch_size, self.seq_length],					self.num_labels )
       a__						=ids_tensor([self.batch_size],					self.num_choices )

     a__						=RemBertConfig(
         vocab_size=self.vocab_size,					hidden_size=self.hidden_size,					input_embedding_size=self.input_embedding_size,					output_embedding_size=self.output_embedding_size,					num_hidden_layers=self.num_hidden_layers,					num_attention_heads=self.num_attention_heads,					intermediate_size=self.intermediate_size,					hidden_act=self.hidden_act,					hidden_dropout_prob=self.hidden_dropout_prob,					attention_probs_dropout_prob=self.attention_probs_dropout_prob,					max_position_embeddings=self.max_position_embeddings,					type_vocab_size=self.type_vocab_size,					is_decoder=lowercase_,					initializer_range=self.initializer_range,					)

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



   def 				_UpperCAmelCase				(       self )							->					Dict:






     """simple docstring"""






     (
         (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),          
     )						=self.prepare_config_and_inputs()

     a__						=True
     a__						=floats_tensor([self.batch_size, self.seq_length, self.hidden_size] )
     a__						=ids_tensor([self.batch_size, self.seq_length],					vocab_size=2 )

     return (
         config,
         input_ids,
         token_type_ids,
         input_mask,
         sequence_labels,
         token_labels,
         choice_labels,
         encoder_hidden_states,
         encoder_attention_mask,
     )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Tuple:






     """simple docstring"""






     a__						=RemBertModel(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_ )
     a__						=model(lowercase_,					token_type_ids=lowercase_ )
     a__						=model(lowercase_ )
     self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					)							->					Dict:






     """simple docstring"""






     a__						=True
     a__						=RemBertModel(lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					encoder_hidden_states=lowercase_,					encoder_attention_mask=lowercase_,					)
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					encoder_hidden_states=lowercase_,					)
     a__						=model(lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_ )
     self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					)							->					int:






     """simple docstring"""






     a__						=RemBertForCausalLM(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					labels=lowercase_ )
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.vocab_size) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Dict:






     """simple docstring"""






     a__						=RemBertForMaskedLM(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					labels=lowercase_ )
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.vocab_size) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					)							->					Optional[Any]:






     """simple docstring"""






     a__						=True
     a__						=True
     a__						=RemBertForCausalLM(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()

     # first forward pass
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					encoder_hidden_states=lowercase_,					encoder_attention_mask=lowercase_,					use_cache=lowercase_,					)
     a__						=outputs.past_key_values

     # create hypothetical multiple next token and extent to next_input_ids
     a__						=ids_tensor((self.batch_size, 3),					config.vocab_size )
     a__						=ids_tensor((self.batch_size, 3),					vocab_size=2 )

     # append to next input_ids and
     a__						=torch.cat([input_ids, next_tokens],					dim=-1 )
     a__						=torch.cat([input_mask, next_mask],					dim=-1 )

     a__						=model(
         lowercase_,					attention_mask=lowercase_,					encoder_hidden_states=lowercase_,					encoder_attention_mask=lowercase_,					output_hidden_states=lowercase_,					)['''hidden_states'''][0]
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					encoder_hidden_states=lowercase_,					encoder_attention_mask=lowercase_,					past_key_values=lowercase_,					output_hidden_states=lowercase_,					)['''hidden_states'''][0]

     # select random slice
     a__						=ids_tensor((1,),					output_from_past.shape[-1] ).item()
     a__						=output_from_no_past[:, -3:, random_slice_idx].detach()
     a__						=output_from_past[:, :, random_slice_idx].detach()

     self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

     # test that outputs are equal for slice
     self.parent.assertTrue(torch.allclose(lowercase_,					lowercase_,					atol=1E-3 ) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					List[Any]:






     """simple docstring"""






     a__						=RemBertForQuestionAnswering(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					start_positions=lowercase_,					end_positions=lowercase_,					)
     self.parent.assertEqual(result.start_logits.shape,					(self.batch_size, self.seq_length) )
     self.parent.assertEqual(result.end_logits.shape,					(self.batch_size, self.seq_length) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					List[str]:






     """simple docstring"""






     a__						=self.num_labels
     a__						=RemBertForSequenceClassification(lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					labels=lowercase_ )
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_labels) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Dict:






     """simple docstring"""






     a__						=self.num_labels
     a__						=RemBertForTokenClassification(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					labels=lowercase_ )
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.num_labels) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Optional[int]:






     """simple docstring"""






     a__						=self.num_choices
     a__						=RemBertForMultipleChoice(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=input_ids.unsqueeze(1 ).expand(-1,					self.num_choices,					-1 ).contiguous()
     a__						=token_type_ids.unsqueeze(1 ).expand(-1,					self.num_choices,					-1 ).contiguous()
     a__						=input_mask.unsqueeze(1 ).expand(-1,					self.num_choices,					-1 ).contiguous()
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					token_type_ids=lowercase_,					labels=lowercase_,					)
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_choices) )



   def 				_UpperCAmelCase				(       self )							->					Dict:






     """simple docstring"""






     a__						=self.prepare_config_and_inputs()
     (
         (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),          
     )						=config_and_inputs
     a__						={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict







@require_torch
class 		__magic_name__    (							__lowerCamelCase	, __lowerCamelCase	, unittest.TestCase							):

   '''simple docstring'''




   lowerCamelCase__				:			str												=     (
       (
           RemBertModel,
           RemBertForMaskedLM,
           RemBertForCausalLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   lowerCamelCase__				:			Optional[Any]												=     (RemBertForCausalLM,) if is_torch_available() else ()
   lowerCamelCase__				:			Dict												=     (
       {
           'feature-extraction': RemBertModel,
           'fill-mask': RemBertForMaskedLM,
           'question-answering': RemBertForQuestionAnswering,
           'text-classification': RemBertForSequenceClassification,
           'text-generation': RemBertForCausalLM,
           'token-classification': RemBertForTokenClassification,
           'zero-shot': RemBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )



   def 				_UpperCAmelCase				(       self )							->					int:






     """simple docstring"""






     a__						=RemBertModelTester(self )
     a__						=ConfigTester(self,					config_class=lowercase_,					hidden_size=37 )



   def 				_UpperCAmelCase				(       self )							->					str:






     """simple docstring"""






     self.config_tester.run_common_tests()



   def 				_UpperCAmelCase				(       self )							->					List[str]:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					int:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
       a__						=type
       self.model_tester.create_and_check_model(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Tuple:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Optional[Any]:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					List[str]:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs_for_decoder()
     self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Tuple:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Dict:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					int:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					List[Any]:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs_for_decoder()
     self.model_tester.create_and_check_model_as_decoder(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					List[Any]:






     """simple docstring"""






     (
         (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),   (
         a__       
     ),          
     )						=self.model_tester.prepare_config_and_inputs_for_decoder()

     a__						=None

     self.model_tester.create_and_check_model_as_decoder(
         lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					)



   @slow
   def 				_UpperCAmelCase				(       self )							->					List[str]:






     """simple docstring"""






     for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       a__						=RemBertModel.from_pretrained(lowercase_ )
       self.assertIsNotNone(lowercase_ )







@require_torch
class 		__magic_name__    (							unittest.TestCase							):

   '''simple docstring'''




   @slow
   def 				_UpperCAmelCase				(       self )							->					str:






     """simple docstring"""






     a__						=RemBertModel.from_pretrained('''google/rembert''' )
     a__						=torch.tensor([[312, 56498, 313, 2125, 313]] )
     a__						=torch.tensor([[0, 0, 0, 1, 1]] )
     with torch.no_grad():
       a__						=model(lowercase_,					token_type_ids=lowercase_,					output_hidden_states=lowercase_ )

     a__						=1152

     a__						=torch.Size((1, 5, hidden_size) )
     self.assertEqual(output['''last_hidden_state'''].shape,					lowercase_ )

     a__						=torch.tensor(
         [
             [
                 [0.0754, -0.2022, 0.1904],
                 [-0.3354, -0.3692, -0.4791],
                 [-0.2314, -0.6729, -0.0749],
                 [-0.0396, -0.3105, -0.4234],
                 [-0.1571, -0.0525, 0.5353],
             ]
         ] )

     # Running on the original tf implementation gives slightly different results here.
     # Not clear why this variations is present
     # TODO: Find reason for discrepancy
     # expected_original_implementation = [[
     #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
     #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
     #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
     #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
     #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
     # ]]

     self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3],					lowercase_,					atol=1E-4 ) )



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase     =      {
    '''configuration_deberta_v2''': ['''DEBERTA_V2_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DebertaV2Config''', '''DebertaV2OnnxConfig'''],
    '''tokenization_deberta_v2''': ['''DebertaV2Tokenizer'''],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase     =      ['''DebertaV2TokenizerFast''']

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase     =      [
           '''TF_DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''TFDebertaV2ForMaskedLM''',
           '''TFDebertaV2ForQuestionAnswering''',
           '''TFDebertaV2ForSequenceClassification''',
           '''TFDebertaV2ForTokenClassification''',
           '''TFDebertaV2Model''',
           '''TFDebertaV2PreTrainedModel''',
       ]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase     =      [
           '''DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''DebertaV2ForMaskedLM''',
           '''DebertaV2ForMultipleChoice''',
           '''DebertaV2ForQuestionAnswering''',
           '''DebertaV2ForSequenceClassification''',
           '''DebertaV2ForTokenClassification''',
           '''DebertaV2Model''',
           '''DebertaV2PreTrainedModel''',
       ]


if TYPE_CHECKING:
       from .configuration_deberta_va import (
           DEBERTA_V2_PRETRAINED_CONFIG_ARCHIVE_MAP,
           DebertaVaConfig,
           DebertaVaOnnxConfig,
       )
       from .tokenization_deberta_va import DebertaVaTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_deberta_va_fast import DebertaVaTokenizerFast

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_deberta_va import (
                  TF_DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFDebertaVaForMaskedLM,
                  TFDebertaVaForQuestionAnswering,
                  TFDebertaVaForSequenceClassification,
                  TFDebertaVaForTokenClassification,
                  TFDebertaVaModel,
                  TFDebertaVaPreTrainedModel,
              )

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_deberta_va import (
                  DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST,
                  DebertaVaForMaskedLM,
                  DebertaVaForMultipleChoice,
                  DebertaVaForQuestionAnswering,
                  DebertaVaForSequenceClassification,
                  DebertaVaForTokenClassification,
                  DebertaVaModel,
                  DebertaVaPreTrainedModel,
              )

else:
       import sys

       lowerCamelCase     =      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
