import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RemBertConfig,
      RemBertForCausalLM,
      RemBertForMaskedLM,
      RemBertForMultipleChoice,
      RemBertForQuestionAnswering,
      RemBertForSequenceClassification,
      RemBertForTokenClassification,
      RemBertModel,
  )
  from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class 				_SCREAMING_SNAKE_CASE:
   def __init__(     self				,  lowerCAmelCase_				,  lowerCAmelCase_=13				,  lowerCAmelCase_=7				,  lowerCAmelCase_=True				,  lowerCAmelCase_=True				,  lowerCAmelCase_=True				,  lowerCAmelCase_=True				,  lowerCAmelCase_=99				,  lowerCAmelCase_=32				,  lowerCAmelCase_=18				,  lowerCAmelCase_=43				,  lowerCAmelCase_=5				,  lowerCAmelCase_=4				,  lowerCAmelCase_=37				,  lowerCAmelCase_="gelu"				,  lowerCAmelCase_=0.1				,  lowerCAmelCase_=0.1				,  lowerCAmelCase_=5_12				,  lowerCAmelCase_=16				,  lowerCAmelCase_=2				,  lowerCAmelCase_=0.02				,  lowerCAmelCase_=3				,  lowerCAmelCase_=4				,  lowerCAmelCase_=None				,  )						->Tuple:
        a	:     Any						      =    parent
        a	:     Optional[int]						      =    batch_size
        a	:     Optional[Any]						      =    seq_length
        a	:     Union[str, Any]						      =    is_training
        a	:     Union[str, Any]						      =    use_input_mask
        a	:     Tuple						      =    use_token_type_ids
        a	:     int						      =    use_labels
        a	:     Dict						      =    vocab_size
        a	:     Union[str, Any]						      =    hidden_size
        a	:     Optional[Any]						      =    input_embedding_size
        a	:     Dict						      =    output_embedding_size
        a	:     Optional[Any]						      =    num_hidden_layers
        a	:     str						      =    num_attention_heads
        a	:     Optional[Any]						      =    intermediate_size
        a	:     int						      =    hidden_act
        a	:     Dict						      =    hidden_dropout_prob
        a	:     List[Any]						      =    attention_probs_dropout_prob
        a	:     List[str]						      =    max_position_embeddings
        a	:     Tuple						      =    type_vocab_size
        a	:     Union[str, Any]						      =    type_sequence_label_size
        a	:     Tuple						      =    initializer_range
        a	:     Dict						      =    num_labels
        a	:     Optional[Any]						      =    num_choices
        a	:     Any						      =    scope
   def _lowercase(     self     )						->List[Any]:
        a	:     Dict						      =    ids_tensor([self.batch_size, self.seq_length]				,  self.vocab_size     )

        a	:     Union[str, Any]						      =    None
        if self.use_input_mask:
             a	:     Optional[Any]						      =    random_attention_mask([self.batch_size, self.seq_length]     )

        a	:     Optional[Any]						      =    None
        if self.use_token_type_ids:
             a	:     List[Any]						      =    ids_tensor([self.batch_size, self.seq_length]				,  self.type_vocab_size     )

        a	:     Any						      =    None
        a	:     Union[str, Any]						      =    None
        a	:     Optional[Any]						      =    None
        if self.use_labels:
             a	:     Union[str, Any]						      =    ids_tensor([self.batch_size]				,  self.type_sequence_label_size     )
             a	:     Dict						      =    ids_tensor([self.batch_size, self.seq_length]				,  self.num_labels     )
             a	:     Tuple						      =    ids_tensor([self.batch_size]				,  self.num_choices     )

        a	:     Any						      =    RemBertConfig(
            vocab_size=self.vocab_size				,  hidden_size=self.hidden_size				,  input_embedding_size=self.input_embedding_size				,  output_embedding_size=self.output_embedding_size				,  num_hidden_layers=self.num_hidden_layers				,  num_attention_heads=self.num_attention_heads				,  intermediate_size=self.intermediate_size				,  hidden_act=self.hidden_act				,  hidden_dropout_prob=self.hidden_dropout_prob				,  attention_probs_dropout_prob=self.attention_probs_dropout_prob				,  max_position_embeddings=self.max_position_embeddings				,  type_vocab_size=self.type_vocab_size				,  is_decoder=lowerCAmelCase_				,  initializer_range=self.initializer_range				,  )

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
   def _lowercase(     self     )						->str:
        (
            (
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,							
        )	:     Dict						      =    self.prepare_config_and_inputs()

        a	:     Optional[int]						      =    True
        a	:     List[str]						      =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
        a	:     Tuple						      =    ids_tensor([self.batch_size, self.seq_length]				,  vocab_size=2     )

        return (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
            encoder_hidden_states,
            encoder_attention_mask,
        )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->Optional[Any]:
        a	:     Optional[Any]						      =    RemBertModel(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     int						      =    model(lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_     )
        a	:     Tuple						      =    model(lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_     )
        a	:     str						      =    model(lowerCAmelCase_     )
        self.parent.assertEqual(result.last_hidden_state.shape				,  (self.batch_size, self.seq_length, self.hidden_size)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  )						->Optional[int]:
        a	:     Tuple						      =    True
        a	:     int						      =    RemBertModel(lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     List[str]						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  encoder_hidden_states=lowerCAmelCase_				,  encoder_attention_mask=lowerCAmelCase_				,  )
        a	:     Optional[Any]						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  encoder_hidden_states=lowerCAmelCase_				,  )
        a	:     int						      =    model(lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_     )
        self.parent.assertEqual(result.last_hidden_state.shape				,  (self.batch_size, self.seq_length, self.hidden_size)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  )						->Optional[Any]:
        a	:     Optional[int]						      =    RemBertForCausalLM(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     List[Any]						      =    model(lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  labels=lowerCAmelCase_     )
        self.parent.assertEqual(result.logits.shape				,  (self.batch_size, self.seq_length, self.vocab_size)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->List[Any]:
        a	:     int						      =    RemBertForMaskedLM(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     List[str]						      =    model(lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  labels=lowerCAmelCase_     )
        self.parent.assertEqual(result.logits.shape				,  (self.batch_size, self.seq_length, self.vocab_size)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  )						->Tuple:
        a	:     Any						      =    True
        a	:     Dict						      =    True
        a	:     Union[str, Any]						      =    RemBertForCausalLM(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()

        # first forward pass
        a	:     Dict						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  encoder_hidden_states=lowerCAmelCase_				,  encoder_attention_mask=lowerCAmelCase_				,  use_cache=lowerCAmelCase_				,  )
        a	:     str						      =    outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        a	:     Union[str, Any]						      =    ids_tensor((self.batch_size, 3)				,  config.vocab_size     )
        a	:     Dict						      =    ids_tensor((self.batch_size, 3)				,  vocab_size=2     )

        # append to next input_ids and
        a	:     str						      =    torch.cat([input_ids, next_tokens]				,  dim=-1     )
        a	:     Optional[Any]						      =    torch.cat([input_mask, next_mask]				,  dim=-1     )

        a	:     List[Any]						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  encoder_hidden_states=lowerCAmelCase_				,  encoder_attention_mask=lowerCAmelCase_				,  output_hidden_states=lowerCAmelCase_				,  )['''hidden_states'''][0]
        a	:     List[Any]						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  encoder_hidden_states=lowerCAmelCase_				,  encoder_attention_mask=lowerCAmelCase_				,  past_key_values=lowerCAmelCase_				,  output_hidden_states=lowerCAmelCase_				,  )['''hidden_states'''][0]

        # select random slice
        a	:     Tuple						      =    ids_tensor((1,)				,  output_from_past.shape[-1]     ).item()
        a	:     Tuple						      =    output_from_no_past[:, -3:, random_slice_idx].detach()
        a	:     List[str]						      =    output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(lowerCAmelCase_				,  lowerCAmelCase_				,  atol=1e-3     )     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->int:
        a	:     Optional[int]						      =    RemBertForQuestionAnswering(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     Union[str, Any]						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  start_positions=lowerCAmelCase_				,  end_positions=lowerCAmelCase_				,  )
        self.parent.assertEqual(result.start_logits.shape				,  (self.batch_size, self.seq_length)     )
        self.parent.assertEqual(result.end_logits.shape				,  (self.batch_size, self.seq_length)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->Optional[int]:
        a	:     Tuple						      =    self.num_labels
        a	:     Tuple						      =    RemBertForSequenceClassification(lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     Dict						      =    model(lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  labels=lowerCAmelCase_     )
        self.parent.assertEqual(result.logits.shape				,  (self.batch_size, self.num_labels)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->int:
        a	:     str						      =    self.num_labels
        a	:     Any						      =    RemBertForTokenClassification(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     List[str]						      =    model(lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  labels=lowerCAmelCase_     )
        self.parent.assertEqual(result.logits.shape				,  (self.batch_size, self.seq_length, self.num_labels)     )
   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->Optional[Any]:
        a	:     List[str]						      =    self.num_choices
        a	:     int						      =    RemBertForMultipleChoice(config=lowerCAmelCase_     )
        model.to(lowerCAmelCase_     )
        model.eval()
        a	:     List[Any]						      =    input_ids.unsqueeze(1     ).expand(-1				,  self.num_choices				,  -1     ).contiguous()
        a	:     Dict						      =    token_type_ids.unsqueeze(1     ).expand(-1				,  self.num_choices				,  -1     ).contiguous()
        a	:     int						      =    input_mask.unsqueeze(1     ).expand(-1				,  self.num_choices				,  -1     ).contiguous()
        a	:     List[str]						      =    model(
            lowerCAmelCase_				,  attention_mask=lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  labels=lowerCAmelCase_				,  )
        self.parent.assertEqual(result.logits.shape				,  (self.batch_size, self.num_choices)     )


   def _lowercase(     self     )						->Optional[Any]:
        a	:     Tuple						      =    self.prepare_config_and_inputs()
        (
            (
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,							
        )	:     Union[str, Any]						      =    config_and_inputs
        a	:     int						      =    {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict






@require_torch
class 				_SCREAMING_SNAKE_CASE( snake_case__							,   snake_case__							,   unittest.TestCase						):
   UpperCamelCase_					:    List[str]			      = (
       (
           RemBertModel,
           RemBertForMaskedLM,
           RemBertForCausalLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   UpperCamelCase_					:    Optional[Any]			      = (RemBertForCausalLM,) if is_torch_available() else ()
   UpperCamelCase_					:    Any			      = (
       {
           'feature-extraction': RemBertModel,
           'fill-mask': RemBertForMaskedLM,
           'question-answering': RemBertForQuestionAnswering,
           'text-classification': RemBertForSequenceClassification,
           'text-generation': RemBertForCausalLM,
           'token-classification': RemBertForTokenClassification,
           'zero-shot': RemBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   def _lowercase(     self     )						->Any:
        a	:     List[str]						      =    RemBertModelTester(self     )
        a	:     Tuple						      =    ConfigTester(self				,  config_class=lowerCAmelCase_				,  hidden_size=37     )
   def _lowercase(     self     )						->Union[str, Any]:
        self.config_tester.run_common_tests()
   def _lowercase(     self     )						->Optional[int]:
        a	:     List[Any]						      =    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowerCAmelCase_     )
   def _lowercase(     self     )						->str:
        a	:     Optional[Any]						      =    self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
             a	:     Any						      =    type
             self.model_tester.create_and_check_model(*lowerCAmelCase_     )
   def _lowercase(     self     )						->List[str]:
        a	:     str						      =    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_     )
   def _lowercase(     self     )						->List[str]:
        a	:     Tuple						      =    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_     )
   def _lowercase(     self     )						->Union[str, Any]:
        a	:     Union[str, Any]						      =    self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCAmelCase_     )
   def _lowercase(     self     )						->Union[str, Any]:
        a	:     str						      =    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_     )
   def _lowercase(     self     )						->List[str]:
        a	:     Any						      =    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_     )
   def _lowercase(     self     )						->str:
        a	:     Any						      =    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_     )
   def _lowercase(     self     )						->Dict:
        a	:     List[str]						      =    self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase_     )
   def _lowercase(     self     )						->int:
        # This regression test was failing with PyTorch < 1.3
        (
            (
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,				(
            a			
        )      ,							
        )	:     int						      =    self.model_tester.prepare_config_and_inputs_for_decoder()

        a	:     List[Any]						      =    None

        self.model_tester.create_and_check_model_as_decoder(
            lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  )


   @slow
   def _lowercase(     self     )						->Tuple:
        for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             a	:     Any						      =    RemBertModel.from_pretrained(lowerCAmelCase_     )
             self.assertIsNotNone(lowerCAmelCase_     )






@require_torch
class 				_SCREAMING_SNAKE_CASE( unittest.TestCase						):
   @slow
   def _lowercase(     self     )						->Optional[Any]:
        # Test exact values at the last hidden layer
        a	:     List[Any]						      =    RemBertModel.from_pretrained('''google/rembert'''     )
        a	:     Optional[int]						      =    torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]     )
        a	:     Tuple						      =    torch.tensor([[0, 0, 0, 1, 1]]     )
        with torch.no_grad():
             a	:     Dict						      =    model(lowerCAmelCase_				,  token_type_ids=lowerCAmelCase_				,  output_hidden_states=lowerCAmelCase_     )

        a	:     str						      =    11_52

        a	:     Any						      =    torch.Size((1, 5, hidden_size)     )
        self.assertEqual(output['''last_hidden_state'''].shape				,  lowerCAmelCase_     )

        a	:     Any						      =    torch.tensor(
            [
                [
                    [0.07_54, -0.20_22, 0.19_04],
                    [-0.33_54, -0.36_92, -0.47_91],
                    [-0.23_14, -0.67_29, -0.07_49],
                    [-0.03_96, -0.31_05, -0.42_34],
                    [-0.15_71, -0.05_25, 0.53_53],
                ]
            ]     )

        # Running on the original tf implementation gives slightly different results here.
        # Not clear why this variations is present
        # TODO: Find reason for discrepancy
        # expected_original_implementation = [[
        #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
        #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
        #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
        #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
        #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
        # ]]

        self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]				,  lowerCAmelCase_				,  atol=1e-4     )     )





from __future__ import annotations

import unittest

from transformers import AlbertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import TF_MODEL_FOR_PRETRAINING_MAPPING
  from transformers.models.albert.modeling_tf_albert import (
      TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFAlbertForMaskedLM,
      TFAlbertForMultipleChoice,
      TFAlbertForPreTraining,
      TFAlbertForQuestionAnswering,
      TFAlbertForSequenceClassification,
      TFAlbertForTokenClassification,
      TFAlbertModel,
  )
class       _UpperCamelCase   :
      def __init__(      self :Optional[int]			,   lowerCamelCase :Tuple			,   lowerCamelCase :Union[str, Any]=13			,   lowerCamelCase :List[str]=7			,   lowerCamelCase :List[Any]=True			,   lowerCamelCase :Any=True			,   lowerCamelCase :Any=True			,   lowerCamelCase :Optional[Any]=True			,   lowerCamelCase :Optional[int]=99			,   lowerCamelCase :Tuple=16			,   lowerCamelCase :Optional[Any]=32			,   lowerCamelCase :Optional[Any]=2			,   lowerCamelCase :List[Any]=4			,   lowerCamelCase :Dict=37			,   lowerCamelCase :Optional[Any]="gelu"			,   lowerCamelCase :Dict=0.1			,   lowerCamelCase :Optional[Any]=0.1			,   lowerCamelCase :Tuple=512			,   lowerCamelCase :Optional[int]=16			,   lowerCamelCase :Any=2			,   lowerCamelCase :Tuple=0.02			,   lowerCamelCase :Union[str, Any]=3			,   lowerCamelCase :str=4			,   lowerCamelCase :Dict=None			,   )  ->      List[Any]:
       UpperCAmelCase__							=    parent
       UpperCAmelCase__							=    13
       UpperCAmelCase__							=    7
       UpperCAmelCase__							=    True
       UpperCAmelCase__							=    True
       UpperCAmelCase__							=    True
       UpperCAmelCase__							=    True
       UpperCAmelCase__							=    99
       UpperCAmelCase__							=    16
       UpperCAmelCase__							=    32
       UpperCAmelCase__							=    2
       UpperCAmelCase__							=    4
       UpperCAmelCase__							=    37
       UpperCAmelCase__							=    "gelu"
       UpperCAmelCase__							=    0.1
       UpperCAmelCase__							=    0.1
       UpperCAmelCase__							=    512
       UpperCAmelCase__							=    16
       UpperCAmelCase__							=    2
       UpperCAmelCase__							=    0.02
       UpperCAmelCase__							=    3
       UpperCAmelCase__							=    4
       UpperCAmelCase__							=    None
      def 						UpperCAmelCase_    (      self :Optional[int]   )  ->      Dict:
       UpperCAmelCase__							=    ids_tensor([self.batch_size, self.seq_length]			,   self.vocab_size   )

       UpperCAmelCase__							=    None
       if self.use_input_mask:
        UpperCAmelCase__							=    random_attention_mask([self.batch_size, self.seq_length]   )

       UpperCAmelCase__							=    None
       if self.use_token_type_ids:
        UpperCAmelCase__							=    ids_tensor([self.batch_size, self.seq_length]			,   self.type_vocab_size   )

       UpperCAmelCase__							=    None
       UpperCAmelCase__							=    None
       UpperCAmelCase__							=    None
       if self.use_labels:
        UpperCAmelCase__							=    ids_tensor([self.batch_size]			,   self.type_sequence_label_size   )
        UpperCAmelCase__							=    ids_tensor([self.batch_size, self.seq_length]			,   self.num_labels   )
        UpperCAmelCase__							=    ids_tensor([self.batch_size]			,   self.num_choices   )

       UpperCAmelCase__							=    AlbertConfig(
           vocab_size=self.vocab_size			,   hidden_size=self.hidden_size			,   embedding_size=self.embedding_size			,   num_hidden_layers=self.num_hidden_layers			,   num_attention_heads=self.num_attention_heads			,   intermediate_size=self.intermediate_size			,   hidden_act=self.hidden_act			,   hidden_dropout_prob=self.hidden_dropout_prob			,   attention_probs_dropout_prob=self.attention_probs_dropout_prob			,   max_position_embeddings=self.max_position_embeddings			,   type_vocab_size=self.type_vocab_size			,   initializer_range=self.initializer_range			,   )

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      def 						UpperCAmelCase_    (      self :Optional[int]			,   lowerCamelCase :str			,   lowerCamelCase :Tuple			,   lowerCamelCase :Tuple			,   lowerCamelCase :List[Any]			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :int			,   lowerCamelCase :str   )  ->      Optional[Any]:
       UpperCAmelCase__							=    TFAlbertModel(config=lowerCamelCase   )
       # inputs = {'input_ids': input_ids,
       #           'attention_mask': input_mask,
       #           'token_type_ids': token_type_ids}
       # sequence_output, pooled_output = model(**inputs)
       UpperCAmelCase__							=    {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase__							=    model(lowerCamelCase   )

       UpperCAmelCase__							=    [input_ids, input_mask]
       UpperCAmelCase__							=    model(lowerCamelCase   )

       UpperCAmelCase__							=    model(lowerCamelCase   )

       self.parent.assertEqual(result.last_hidden_state.shape			,   (self.batch_size, self.seq_length, self.hidden_size)   )
       self.parent.assertEqual(result.pooler_output.shape			,   (self.batch_size, self.hidden_size)   )
      def 						UpperCAmelCase_    (      self :Optional[Any]			,   lowerCamelCase :Any			,   lowerCamelCase :List[Any]			,   lowerCamelCase :Any			,   lowerCamelCase :List[str]			,   lowerCamelCase :Optional[int]			,   lowerCamelCase :List[Any]			,   lowerCamelCase :Tuple   )  ->      Optional[int]:
       UpperCAmelCase__							=    self.num_labels
       UpperCAmelCase__							=    TFAlbertForPreTraining(config=lowerCamelCase   )
       UpperCAmelCase__							=    {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase__							=    model(lowerCamelCase   )
       self.parent.assertEqual(result.prediction_logits.shape			,   (self.batch_size, self.seq_length, self.vocab_size)   )
       self.parent.assertEqual(result.sop_logits.shape			,   (self.batch_size, self.num_labels)   )
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :Dict			,   lowerCamelCase :List[Any]			,   lowerCamelCase :List[str]			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :Optional[int]			,   lowerCamelCase :Tuple			,   lowerCamelCase :Any   )  ->      Optional[Any]:
       UpperCAmelCase__							=    TFAlbertForMaskedLM(config=lowerCamelCase   )
       UpperCAmelCase__							=    {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase__							=    model(lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape			,   (self.batch_size, self.seq_length, self.vocab_size)   )
      def 						UpperCAmelCase_    (      self :Tuple			,   lowerCamelCase :str			,   lowerCamelCase :Dict			,   lowerCamelCase :Dict			,   lowerCamelCase :Optional[Any]			,   lowerCamelCase :List[Any]			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :Optional[Any]   )  ->      str:
       UpperCAmelCase__							=    self.num_labels
       UpperCAmelCase__							=    TFAlbertForSequenceClassification(config=lowerCamelCase   )
       UpperCAmelCase__							=    {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase__							=    model(lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape			,   (self.batch_size, self.num_labels)   )
      def 						UpperCAmelCase_    (      self :Dict			,   lowerCamelCase :Optional[Any]			,   lowerCamelCase :str			,   lowerCamelCase :Dict			,   lowerCamelCase :List[Any]			,   lowerCamelCase :List[str]			,   lowerCamelCase :int			,   lowerCamelCase :Optional[Any]   )  ->      Dict:
       UpperCAmelCase__							=    TFAlbertForQuestionAnswering(config=lowerCamelCase   )
       UpperCAmelCase__							=    {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase__							=    model(lowerCamelCase   )
       self.parent.assertEqual(result.start_logits.shape			,   (self.batch_size, self.seq_length)   )
       self.parent.assertEqual(result.end_logits.shape			,   (self.batch_size, self.seq_length)   )
      def 						UpperCAmelCase_    (      self :Optional[int]			,   lowerCamelCase :Optional[Any]			,   lowerCamelCase :Dict			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :Tuple			,   lowerCamelCase :List[Any]			,   lowerCamelCase :Union[str, Any]   )  ->      int:
       UpperCAmelCase__							=    self.num_choices
       UpperCAmelCase__							=    TFAlbertForMultipleChoice(config=lowerCamelCase   )
       UpperCAmelCase__							=    tf.tile(tf.expand_dims(lowerCamelCase			,   1   )			,   (1, self.num_choices, 1)   )
       UpperCAmelCase__							=    tf.tile(tf.expand_dims(lowerCamelCase			,   1   )			,   (1, self.num_choices, 1)   )
       UpperCAmelCase__							=    tf.tile(tf.expand_dims(lowerCamelCase			,   1   )			,   (1, self.num_choices, 1)   )
       UpperCAmelCase__							=    {
           "input_ids": multiple_choice_inputs_ids,
           "attention_mask": multiple_choice_input_mask,
           "token_type_ids": multiple_choice_token_type_ids,
       }
       UpperCAmelCase__							=    model(lowerCamelCase   )
       self.parent.assertListEqual(list(result["logits"].shape   )			,   [self.batch_size, self.num_choices]   )
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :List[str]			,   lowerCamelCase :int			,   lowerCamelCase :int			,   lowerCamelCase :Union[str, Any]			,   lowerCamelCase :Dict			,   lowerCamelCase :Dict   )  ->      List[str]:
       UpperCAmelCase__							=    self.num_labels
       UpperCAmelCase__							=    TFAlbertForTokenClassification(config=lowerCamelCase   )
       UpperCAmelCase__							=    {
           "input_ids": input_ids,
           "attention_mask": input_mask,
           "token_type_ids": token_type_ids,
       }
       UpperCAmelCase__							=    model(lowerCamelCase   )
       self.parent.assertListEqual(list(result["logits"].shape   )			,   [self.batch_size, self.seq_length, self.num_labels]   )







      def 						UpperCAmelCase_    (      self :str   )  ->      Any:
       UpperCAmelCase__							=    self.prepare_config_and_inputs()
       (
           (
           UpperCAmelCase__
       )			,			(
           UpperCAmelCase__
       )			,			(
           UpperCAmelCase__
       )			,			(
           UpperCAmelCase__
       )			,			(
           UpperCAmelCase__
       )			,			(
           UpperCAmelCase__
       )			,			(
           UpperCAmelCase__
       )			,			
       )							=    config_and_inputs
       UpperCAmelCase__							=    {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
       return config, inputs_dict



@require_tf
class       _UpperCamelCase   (					lowerCAmelCase						,							lowerCAmelCase						,							unittest.TestCase		):
      UpperCAmelCase_          =		(
          (
              TFAlbertModel,
              TFAlbertForPreTraining,
              TFAlbertForMaskedLM,
              TFAlbertForSequenceClassification,
              TFAlbertForQuestionAnswering,
              TFAlbertForTokenClassification,
              TFAlbertForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      UpperCAmelCase_          =		(
          {
              """feature-extraction""": TFAlbertModel,
              """fill-mask""": TFAlbertForMaskedLM,
              """question-answering""": TFAlbertForQuestionAnswering,
              """text-classification""": TFAlbertForSequenceClassification,
              """token-classification""": TFAlbertForTokenClassification,
              """zero-shot""": TFAlbertForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      UpperCAmelCase_          =		False
      UpperCAmelCase_          =		False
      def 						UpperCAmelCase_    (      self :str			,   lowerCamelCase :Any			,   lowerCamelCase :Optional[int]			,   lowerCamelCase :Optional[int]=False   )  ->      str:
       UpperCAmelCase__							=    super()._prepare_for_class(lowerCamelCase			,   lowerCamelCase			,   return_labels=lowerCamelCase   )

       if return_labels:
        if model_class in get_values(lowerCamelCase   ):
         UpperCAmelCase__							=    tf.zeros(self.model_tester.batch_size			,   dtype=tf.intaa   )

       return inputs_dict
      def 						UpperCAmelCase_    (      self :List[Any]   )  ->      List[Any]:
       UpperCAmelCase__							=    TFAlbertModelTester(self   )
       UpperCAmelCase__							=    ConfigTester(self			,   config_class=lowerCamelCase			,   hidden_size=37   )
      def 						UpperCAmelCase_    (      self :Dict   )  ->      Union[str, Any]:
       self.config_tester.run_common_tests()
      def 						UpperCAmelCase_    (      self :Optional[Any]   )  ->      Tuple:
       UpperCAmelCase__							=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_albert_model(*lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :List[Any]   )  ->      Dict:
       UpperCAmelCase__							=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_albert_for_pretraining(*lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :List[Any]   )  ->      str:
       UpperCAmelCase__							=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_albert_for_masked_lm(*lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :Dict   )  ->      int:
       UpperCAmelCase__							=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_albert_for_multiple_choice(*lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :Union[str, Any]   )  ->      Dict:
       UpperCAmelCase__							=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_albert_for_sequence_classification(*lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :str   )  ->      int:
       UpperCAmelCase__							=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_albert_for_question_answering(*lowerCamelCase   )







      @slow
      def 						UpperCAmelCase_    (      self :int   )  ->      str:
       for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        UpperCAmelCase__							=    TFAlbertModel.from_pretrained(lowerCamelCase   )
        self.assertIsNotNone(lowerCamelCase   )



@require_tf
class       _UpperCamelCase   (					unittest.TestCase		):
      @slow
      def 						UpperCAmelCase_    (      self :List[str]   )  ->      List[str]:
       UpperCAmelCase__							=    TFAlbertForPreTraining.from_pretrained("albert-base-v2"   )
       UpperCAmelCase__							=    tf.constant([[0, 1, 2, 3, 4, 5]]   )
       UpperCAmelCase__							=    model(lowerCamelCase   )[0]

       UpperCAmelCase__							=    [1, 6, 3_0000]
       self.assertEqual(output.shape			,   lowerCamelCase   )

       UpperCAmelCase__							=    tf.constant(
           [
               [
                   [4.59_56_68, 0.74_46_27_54, -1.81_81_47],
                   [4.5_95_43_47, 0.7_45_41_84, -1.8_18_82_58],
                   [4.5_95_49_05, 0.7_44_82_35, -1.8_18_23_16],
               ]
           ]   )
       tf.debugging.assert_near(output[:, :3, :3]			,   lowerCamelCase			,   atol=1e-4   )





