
from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          TFRoFormerForCausalLM,
          TFRoFormerForMaskedLM,
          TFRoFormerForMultipleChoice,
          TFRoFormerForQuestionAnswering,
          TFRoFormerForSequenceClassification,
          TFRoFormerForTokenClassification,
          TFRoFormerModel,
      )
      from transformers.models.roformer.modeling_tf_roformer import (
          TFRoFormerSelfAttention,
          TFRoFormerSinusoidalPositionalEmbedding,
      )



class 		UpperCamelCase__  :
   def __init__(self :					Union[str, Any]					,				snake_case_ :					int					,				snake_case_ :					int=1_3					,				snake_case_ :					Dict=7					,				snake_case_ :					List[str]=True					,				snake_case_ :					Dict=True					,				snake_case_ :					Union[str, Any]=True					,				snake_case_ :					List[Any]=True					,				snake_case_ :					Tuple=9_9					,				snake_case_ :					Tuple=3_2					,				snake_case_ :					List[Any]=2					,				snake_case_ :					Tuple=4					,				snake_case_ :					str=3_7					,				snake_case_ :					Union[str, Any]="gelu"					,				snake_case_ :					List[Any]=0.1					,				snake_case_ :					str=0.1					,				snake_case_ :					int=5_1_2					,				snake_case_ :					List[Any]=1_6					,				snake_case_ :					Tuple=2					,				snake_case_ :					Optional[Any]=0.02					,				snake_case_ :					Optional[Any]=3					,				snake_case_ :					Optional[int]=4					,				snake_case_ :					int=None					,				):
    __a :  Optional[int]						=  parent
    __a :  int						=  1_3
    __a :  str						=  7
    __a :  Union[str, Any]						=  True
    __a :  int						=  True
    __a :  Tuple						=  True
    __a :  Tuple						=  True
    __a :  List[Any]						=  9_9
    __a :  Optional[int]						=  3_2
    __a :  List[Any]						=  2
    __a :  Optional[Any]						=  4
    __a :  Tuple						=  3_7
    __a :  Dict						=  '''gelu'''
    __a :  int						=  0.1
    __a :  List[Any]						=  0.1
    __a :  Union[str, Any]						=  5_1_2
    __a :  Tuple						=  1_6
    __a :  Optional[Any]						=  2
    __a :  Dict						=  0.02
    __a :  List[Any]						=  3
    __a :  Any						=  4
    __a :  List[str]						=  None
   def       lowerCAmelCase			(self :					List[Any]   ):
    __a :  List[Any]						=  ids_tensor([self.batch_size, self.seq_length]					,				self.vocab_size   )

    __a :  List[str]						=  None
    if self.use_input_mask:
     __a :  List[Any]						=  random_attention_mask([self.batch_size, self.seq_length]   )

    __a :  Optional[int]						=  None
    if self.use_token_type_ids:
     __a :  Optional[Any]						=  ids_tensor([self.batch_size, self.seq_length]					,				self.type_vocab_size   )

    __a :  List[str]						=  None
    __a :  List[Any]						=  None
    __a :  Optional[Any]						=  None
    if self.use_labels:
     __a :  Optional[Any]						=  ids_tensor([self.batch_size]					,				self.type_sequence_label_size   )
     __a :  List[str]						=  ids_tensor([self.batch_size, self.seq_length]					,				self.num_labels   )
     __a :  Optional[int]						=  ids_tensor([self.batch_size]					,				self.num_choices   )

    __a :  Tuple						=  RoFormerConfig(
        vocab_size=self.vocab_size					,				hidden_size=self.hidden_size					,				num_hidden_layers=self.num_hidden_layers					,				num_attention_heads=self.num_attention_heads					,				intermediate_size=self.intermediate_size					,				hidden_act=self.hidden_act					,				hidden_dropout_prob=self.hidden_dropout_prob					,				attention_probs_dropout_prob=self.attention_probs_dropout_prob					,				max_position_embeddings=self.max_position_embeddings					,				type_vocab_size=self.type_vocab_size					,				initializer_range=self.initializer_range					,				return_dict=snake_case_					,				)

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
   def       lowerCAmelCase			(self :					Dict					,				snake_case_ :					Tuple					,				snake_case_ :					int					,				snake_case_ :					List[Any]					,				snake_case_ :					Any					,				snake_case_ :					str					,				snake_case_ :					Dict					,				snake_case_ :					List[Any]   ):
    __a :  Union[str, Any]						=  TFRoFormerModel(config=snake_case_   )
    __a :  int						=  {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

    __a :  List[Any]						=  [input_ids, input_mask]
    __a :  str						=  model(snake_case_   )

    __a :  Any						=  model(snake_case_   )

    self.parent.assertEqual(result.last_hidden_state.shape					,				(self.batch_size, self.seq_length, self.hidden_size)   )
   def       lowerCAmelCase			(self :					Any					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					int					,				snake_case_ :					List[str]					,				snake_case_ :					int					,				snake_case_ :					Optional[int]					,				snake_case_ :					Dict   ):
    __a :  Union[str, Any]						=  True
    __a :  Dict						=  TFRoFormerForCausalLM(config=snake_case_   )
    __a :  Optional[Any]						=  {
        '''input_ids''': input_ids,
        '''attention_mask''': input_mask,
        '''token_type_ids''': token_type_ids,
    }
    __a :  Union[str, Any]						=  model(snake_case_   )['''logits''']
    self.parent.assertListEqual(
        list(prediction_scores.numpy().shape   )					,				[self.batch_size, self.seq_length, self.vocab_size]   )
   def       lowerCAmelCase			(self :					Any					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					int					,				snake_case_ :					List[Any]					,				snake_case_ :					Dict					,				snake_case_ :					int					,				snake_case_ :					Tuple   ):
    __a :  Optional[int]						=  TFRoFormerForMaskedLM(config=snake_case_   )
    __a :  List[str]						=  {
        '''input_ids''': input_ids,
        '''attention_mask''': input_mask,
        '''token_type_ids''': token_type_ids,
    }
    __a :  Optional[Any]						=  model(snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.seq_length, self.vocab_size)   )
   def       lowerCAmelCase			(self :					Dict					,				snake_case_ :					str					,				snake_case_ :					Tuple					,				snake_case_ :					int					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Optional[int]					,				snake_case_ :					Union[str, Any]   ):
    __a :  List[Any]						=  self.num_labels
    __a :  Optional[Any]						=  TFRoFormerForSequenceClassification(config=snake_case_   )
    __a :  Any						=  {
        '''input_ids''': input_ids,
        '''attention_mask''': input_mask,
        '''token_type_ids''': token_type_ids,
    }

    __a :  List[Any]						=  model(snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.num_labels)   )
   def       lowerCAmelCase			(self :					int					,				snake_case_ :					Dict					,				snake_case_ :					List[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Optional[int]					,				snake_case_ :					List[Any]					,				snake_case_ :					List[str]					,				snake_case_ :					Optional[Any]   ):
    __a :  List[Any]						=  self.num_choices
    __a :  Any						=  TFRoFormerForMultipleChoice(config=snake_case_   )
    __a :  Any						=  tf.tile(tf.expand_dims(snake_case_					,				1   )					,				(1, self.num_choices, 1)   )
    __a :  Tuple						=  tf.tile(tf.expand_dims(snake_case_					,				1   )					,				(1, self.num_choices, 1)   )
    __a :  int						=  tf.tile(tf.expand_dims(snake_case_					,				1   )					,				(1, self.num_choices, 1)   )
    __a :  Dict						=  {
        '''input_ids''': multiple_choice_inputs_ids,
        '''attention_mask''': multiple_choice_input_mask,
        '''token_type_ids''': multiple_choice_token_type_ids,
    }
    __a :  Any						=  model(snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.num_choices)   )
   def       lowerCAmelCase			(self :					Any					,				snake_case_ :					int					,				snake_case_ :					List[Any]					,				snake_case_ :					List[str]					,				snake_case_ :					Tuple					,				snake_case_ :					List[str]					,				snake_case_ :					Union[str, Any]					,				snake_case_ :					int   ):
    __a :  Any						=  self.num_labels
    __a :  str						=  TFRoFormerForTokenClassification(config=snake_case_   )
    __a :  Any						=  {
        '''input_ids''': input_ids,
        '''attention_mask''': input_mask,
        '''token_type_ids''': token_type_ids,
    }
    __a :  Optional[Any]						=  model(snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.seq_length, self.num_labels)   )
   def       lowerCAmelCase			(self :					Dict					,				snake_case_ :					Tuple					,				snake_case_ :					List[Any]					,				snake_case_ :					List[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					str					,				snake_case_ :					Tuple					,				snake_case_ :					List[Any]   ):
    __a :  Optional[int]						=  TFRoFormerForQuestionAnswering(config=snake_case_   )
    __a :  int						=  {
        '''input_ids''': input_ids,
        '''attention_mask''': input_mask,
        '''token_type_ids''': token_type_ids,
    }

    __a :  List[Any]						=  model(snake_case_   )
    self.parent.assertEqual(result.start_logits.shape					,				(self.batch_size, self.seq_length)   )
    self.parent.assertEqual(result.end_logits.shape					,				(self.batch_size, self.seq_length)   )







   def       lowerCAmelCase			(self :					int   ):
    __a :  List[str]						=  self.prepare_config_and_inputs()
    (
        (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       
    ) :  List[Any]						=  config_and_inputs
    __a :  Optional[int]						=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
    return config, inputs_dict





@require_tf
class 		UpperCamelCase__  (       __lowercase    ,__lowercase    ,unittest.TestCase    ):
   _SCREAMING_SNAKE_CASE      :							Optional[Any]	       =      (
       (
           TFRoFormerModel,
           TFRoFormerForCausalLM,
           TFRoFormerForMaskedLM,
           TFRoFormerForQuestionAnswering,
           TFRoFormerForSequenceClassification,
           TFRoFormerForTokenClassification,
           TFRoFormerForMultipleChoice,
       )
       if is_tf_available()
       else ()
   )
   _SCREAMING_SNAKE_CASE      :							int	       =      (
       {
           "feature-extraction": TFRoFormerModel,
           "fill-mask": TFRoFormerForMaskedLM,
           "question-answering": TFRoFormerForQuestionAnswering,
           "text-classification": TFRoFormerForSequenceClassification,
           "text-generation": TFRoFormerForCausalLM,
           "token-classification": TFRoFormerForTokenClassification,
           "zero-shot": TFRoFormerForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )

   _SCREAMING_SNAKE_CASE      :							Any	       =      False
   _SCREAMING_SNAKE_CASE      :							Optional[int]	       =      False
   def       lowerCAmelCase			(self :					List[str]					,				snake_case_ :					Any					,				snake_case_ :					Any					,				snake_case_ :					Tuple					,				snake_case_ :					Any					,				snake_case_ :					Optional[int]   ):
    if pipeline_test_casse_name == "TextGenerationPipelineTests":
     return True

    return False
   def       lowerCAmelCase			(self :					Any   ):
    __a :  List[Any]						=  TFRoFormerModelTester(self   )
    __a :  int						=  ConfigTester(self					,				config_class=snake_case_					,				hidden_size=3_7   )
   def       lowerCAmelCase			(self :					Optional[int]   ):
    self.config_tester.run_common_tests()
   def       lowerCAmelCase			(self :					str   ):
    __a :  List[str]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*snake_case_   )
   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  Union[str, Any]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*snake_case_   )
   def       lowerCAmelCase			(self :					Union[str, Any]   ):
    __a :  str						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_lm_head(*snake_case_   )
   def       lowerCAmelCase			(self :					Dict   ):
    __a :  Dict						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*snake_case_   )
   def       lowerCAmelCase			(self :					str   ):
    __a :  int						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*snake_case_   )
   def       lowerCAmelCase			(self :					Dict   ):
    __a :  Tuple						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*snake_case_   )
   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  Tuple						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*snake_case_   )







   @slow
   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  Any						=  TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''   )
    self.assertIsNotNone(snake_case_   )





@require_tf
class 		UpperCamelCase__  (       unittest.TestCase    ):
   @slow
   def       lowerCAmelCase			(self :					Union[str, Any]   ):
    __a :  Optional[Any]						=  TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''   )
    __a :  Optional[int]						=  tf.constant([[0, 1, 2, 3, 4, 5]]   )
    __a :  Dict						=  model(snake_case_   )[0]

    # TODO Replace vocab size
    __a :  Dict						=  5_0_0_0_0

    __a :  List[str]						=  [1, 6, vocab_size]
    self.assertEqual(output.shape					,				snake_case_   )

    print(output[:, :3, :3]   )

    # TODO Replace values below with what was printed above.
    __a :  Tuple						=  tf.constant(
        [
            [
                [-0.1205_3341, -1.026_4901, 0.2922_1946],
                [-1.513_3783, 0.19_7433, 0.1519_0607],
                [-5.013_5403, -3.90_0256, -0.8403_8764],
            ]
        ]   )
    tf.debugging.assert_near(output[:, :3, :3]					,				snake_case_					,				atol=1E-4   )





@require_tf
class 		UpperCamelCase__  (       unittest.TestCase    ):
   _SCREAMING_SNAKE_CASE      :							Any	       =      1e-4
   def       lowerCAmelCase			(self :					Optional[Any]   ):
    __a :  int						=  tf.constant([[4, 1_0]]   )
    __a :  str						=  TFRoFormerSinusoidalPositionalEmbedding(num_positions=6					,				embedding_dim=6   )

    __a :  Optional[int]						=  emba(input_ids.shape   )
    __a :  List[str]						=  tf.constant(
        [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]   )

    tf.debugging.assert_near(snake_case_					,				snake_case_					,				atol=self.tolerance   )







   def       lowerCAmelCase			(self :					List[str]   ):
    __a :  List[Any]						=  tf.constant(
        [
            [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
            [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
            [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
        ]   )
    __a :  str						=  TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_1_2					,				embedding_dim=5_1_2   )
    emba([2, 1_6, 5_1_2]   )
    __a :  List[Any]						=  emba.weight[:3, :5]

    tf.debugging.assert_near(snake_case_					,				snake_case_					,				atol=self.tolerance   )





@require_tf
class 		UpperCamelCase__  (       unittest.TestCase    ):
   _SCREAMING_SNAKE_CASE      :							List[Any]	       =      1e-4
   def       lowerCAmelCase			(self :					str   ):
    # 2,12,16,64
    __a :  Optional[Any]						=  tf.reshape(tf.range(2 * 1_2 * 1_6 * 6_4					,				dtype=tf.floataa   )					,				shape=(2, 1_2, 1_6, 6_4)   ) / 1_0_0

    __a :  str						=  -tf.reshape(tf.range(2 * 1_2 * 1_6 * 6_4					,				dtype=tf.floataa   )					,				shape=(2, 1_2, 1_6, 6_4)   ) / 1_0_0

    __a :  str						=  TFRoFormerSinusoidalPositionalEmbedding(num_positions=3_2					,				embedding_dim=6_4   )
    __a :  Optional[Any]						=  embed_positions([2, 1_6, 7_6_8]   )[None, None, :, :]

    __a				,       __a :  Tuple						=  TFRoFormerSelfAttention.apply_rotary_position_embeddings(
        snake_case_					,				snake_case_					,				snake_case_   )

    __a :  Optional[Any]						=  tf.constant(
        [
            [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
            [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
            [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
            [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
            [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
            [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
        ]   )
    __a :  int						=  tf.constant(
        [
            [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
            [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
            [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
            [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
            [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
            [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
        ]   )

    tf.debugging.assert_near(query_layer[0, 0, :6, :8]					,				snake_case_					,				atol=self.tolerance   )
    tf.debugging.assert_near(key_layer[0, 0, :6, :8]					,				snake_case_					,				atol=self.tolerance   )


import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
          MobileBertModel,
      )



class 		UpperCamelCase__  :
   def __init__(self :					List[Any]					,				snake_case_ :					int					,				snake_case_ :					List[str]=1_3					,				snake_case_ :					Tuple=7					,				snake_case_ :					List[Any]=True					,				snake_case_ :					List[Any]=True					,				snake_case_ :					Dict=True					,				snake_case_ :					Optional[int]=True					,				snake_case_ :					str=9_9					,				snake_case_ :					Dict=6_4					,				snake_case_ :					Any=3_2					,				snake_case_ :					str=5					,				snake_case_ :					int=4					,				snake_case_ :					List[Any]=3_7					,				snake_case_ :					Any="gelu"					,				snake_case_ :					Dict=0.1					,				snake_case_ :					List[str]=0.1					,				snake_case_ :					str=5_1_2					,				snake_case_ :					Any=1_6					,				snake_case_ :					str=2					,				snake_case_ :					int=0.02					,				snake_case_ :					Union[str, Any]=3					,				snake_case_ :					Optional[Any]=4					,				snake_case_ :					List[Any]=None					,				):
    __a :  Any						=  parent
    __a :  Optional[int]						=  batch_size
    __a :  Any						=  seq_length
    __a :  int						=  is_training
    __a :  Optional[int]						=  use_input_mask
    __a :  List[Any]						=  use_token_type_ids
    __a :  Dict						=  use_labels
    __a :  Tuple						=  vocab_size
    __a :  str						=  hidden_size
    __a :  List[Any]						=  embedding_size
    __a :  List[Any]						=  num_hidden_layers
    __a :  str						=  num_attention_heads
    __a :  str						=  intermediate_size
    __a :  Union[str, Any]						=  hidden_act
    __a :  Optional[Any]						=  hidden_dropout_prob
    __a :  Tuple						=  attention_probs_dropout_prob
    __a :  Union[str, Any]						=  max_position_embeddings
    __a :  Any						=  type_vocab_size
    __a :  int						=  type_sequence_label_size
    __a :  int						=  initializer_range
    __a :  int						=  num_labels
    __a :  Union[str, Any]						=  num_choices
    __a :  Dict						=  scope
   def       lowerCAmelCase			(self :					str   ):
    __a :  int						=  ids_tensor([self.batch_size, self.seq_length]					,				self.vocab_size   )

    __a :  List[Any]						=  None
    if self.use_input_mask:
     __a :  Dict						=  random_attention_mask([self.batch_size, self.seq_length]   )

    __a :  Optional[Any]						=  None
    if self.use_token_type_ids:
     __a :  List[Any]						=  ids_tensor([self.batch_size, self.seq_length]					,				self.type_vocab_size   )

    __a :  Dict						=  None
    __a :  List[str]						=  None
    __a :  Optional[Any]						=  None
    if self.use_labels:
     __a :  str						=  ids_tensor([self.batch_size]					,				self.type_sequence_label_size   )
     __a :  Dict						=  ids_tensor([self.batch_size, self.seq_length]					,				self.num_labels   )
     __a :  Union[str, Any]						=  ids_tensor([self.batch_size]					,				self.num_choices   )

    __a :  Optional[int]						=  self.get_config()

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
   def       lowerCAmelCase			(self :					int   ):
    return MobileBertConfig(
        vocab_size=self.vocab_size					,				hidden_size=self.hidden_size					,				num_hidden_layers=self.num_hidden_layers					,				num_attention_heads=self.num_attention_heads					,				intermediate_size=self.intermediate_size					,				embedding_size=self.embedding_size					,				hidden_act=self.hidden_act					,				hidden_dropout_prob=self.hidden_dropout_prob					,				attention_probs_dropout_prob=self.attention_probs_dropout_prob					,				max_position_embeddings=self.max_position_embeddings					,				type_vocab_size=self.type_vocab_size					,				is_decoder=snake_case_					,				initializer_range=self.initializer_range					,				)
   def       lowerCAmelCase			(self :					str					,				snake_case_ :					Tuple					,				snake_case_ :					List[str]					,				snake_case_ :					int					,				snake_case_ :					int					,				snake_case_ :					Any					,				snake_case_ :					List[Any]					,				snake_case_ :					Any   ):
    __a :  Any						=  MobileBertModel(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  List[str]						=  model(snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_   )
    __a :  Optional[Any]						=  model(snake_case_					,				token_type_ids=snake_case_   )
    __a :  Optional[Any]						=  model(snake_case_   )

    self.parent.assertEqual(result.last_hidden_state.shape					,				(self.batch_size, self.seq_length, self.hidden_size)   )
    self.parent.assertEqual(result.pooler_output.shape					,				(self.batch_size, self.hidden_size)   )
   def       lowerCAmelCase			(self :					Any					,				snake_case_ :					Dict					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Tuple					,				snake_case_ :					Dict					,				snake_case_ :					Tuple					,				snake_case_ :					str					,				snake_case_ :					List[Any]   ):
    __a :  str						=  MobileBertForMaskedLM(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  Tuple						=  model(snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				labels=snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.seq_length, self.vocab_size)   )
   def       lowerCAmelCase			(self :					Tuple					,				snake_case_ :					Any					,				snake_case_ :					Dict					,				snake_case_ :					List[Any]					,				snake_case_ :					Any					,				snake_case_ :					Union[str, Any]					,				snake_case_ :					Optional[int]					,				snake_case_ :					Dict   ):
    __a :  Optional[Any]						=  MobileBertForNextSentencePrediction(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  int						=  model(
        snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				labels=snake_case_					,				)
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, 2)   )
   def       lowerCAmelCase			(self :					Any					,				snake_case_ :					Dict					,				snake_case_ :					Optional[int]					,				snake_case_ :					Any					,				snake_case_ :					Any					,				snake_case_ :					Union[str, Any]					,				snake_case_ :					Dict					,				snake_case_ :					Optional[Any]   ):
    __a :  str						=  MobileBertForPreTraining(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  Union[str, Any]						=  model(
        snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				labels=snake_case_					,				next_sentence_label=snake_case_					,				)
    self.parent.assertEqual(result.prediction_logits.shape					,				(self.batch_size, self.seq_length, self.vocab_size)   )
    self.parent.assertEqual(result.seq_relationship_logits.shape					,				(self.batch_size, 2)   )
   def       lowerCAmelCase			(self :					Dict					,				snake_case_ :					Tuple					,				snake_case_ :					Any					,				snake_case_ :					Dict					,				snake_case_ :					int					,				snake_case_ :					int					,				snake_case_ :					str					,				snake_case_ :					str   ):
    __a :  str						=  MobileBertForQuestionAnswering(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  Optional[Any]						=  model(
        snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				start_positions=snake_case_					,				end_positions=snake_case_					,				)
    self.parent.assertEqual(result.start_logits.shape					,				(self.batch_size, self.seq_length)   )
    self.parent.assertEqual(result.end_logits.shape					,				(self.batch_size, self.seq_length)   )
   def       lowerCAmelCase			(self :					Optional[int]					,				snake_case_ :					Any					,				snake_case_ :					List[str]					,				snake_case_ :					List[Any]					,				snake_case_ :					str					,				snake_case_ :					Any					,				snake_case_ :					Tuple					,				snake_case_ :					Optional[int]   ):
    __a :  Any						=  self.num_labels
    __a :  Union[str, Any]						=  MobileBertForSequenceClassification(snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  Tuple						=  model(snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				labels=snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.num_labels)   )
   def       lowerCAmelCase			(self :					List[Any]					,				snake_case_ :					Optional[int]					,				snake_case_ :					str					,				snake_case_ :					Dict					,				snake_case_ :					List[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Dict					,				snake_case_ :					Optional[int]   ):
    __a :  Union[str, Any]						=  self.num_labels
    __a :  str						=  MobileBertForTokenClassification(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  Any						=  model(snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				labels=snake_case_   )
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.seq_length, self.num_labels)   )
   def       lowerCAmelCase			(self :					Union[str, Any]					,				snake_case_ :					List[str]					,				snake_case_ :					Any					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Optional[Any]					,				snake_case_ :					Any					,				snake_case_ :					Dict					,				snake_case_ :					Union[str, Any]   ):
    __a :  Union[str, Any]						=  self.num_choices
    __a :  List[str]						=  MobileBertForMultipleChoice(config=snake_case_   )
    model.to(snake_case_   )
    model.eval()
    __a :  List[Any]						=  input_ids.unsqueeze(1   ).expand(-1					,				self.num_choices					,				-1   ).contiguous()
    __a :  int						=  token_type_ids.unsqueeze(1   ).expand(-1					,				self.num_choices					,				-1   ).contiguous()
    __a :  str						=  input_mask.unsqueeze(1   ).expand(-1					,				self.num_choices					,				-1   ).contiguous()
    __a :  Any						=  model(
        snake_case_					,				attention_mask=snake_case_					,				token_type_ids=snake_case_					,				labels=snake_case_					,				)
    self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.num_choices)   )







   def       lowerCAmelCase			(self :					Optional[Any]   ):
    __a :  Optional[Any]						=  self.prepare_config_and_inputs()
    (
        (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       (
        __a
    )				,       
    ) :  int						=  config_and_inputs
    __a :  Union[str, Any]						=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
    return config, inputs_dict





@require_torch
class 		UpperCamelCase__  (       __lowercase    ,__lowercase    ,unittest.TestCase    ):
   _SCREAMING_SNAKE_CASE      :							str	       =      (
       (
           MobileBertModel,
           MobileBertForMaskedLM,
           MobileBertForMultipleChoice,
           MobileBertForNextSentencePrediction,
           MobileBertForPreTraining,
           MobileBertForQuestionAnswering,
           MobileBertForSequenceClassification,
           MobileBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   _SCREAMING_SNAKE_CASE      :							Any	       =      (
       {
           "feature-extraction": MobileBertModel,
           "fill-mask": MobileBertForMaskedLM,
           "question-answering": MobileBertForQuestionAnswering,
           "text-classification": MobileBertForSequenceClassification,
           "token-classification": MobileBertForTokenClassification,
           "zero-shot": MobileBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   _SCREAMING_SNAKE_CASE      :							Union[str, Any]	       =      True
   def       lowerCAmelCase			(self :					str					,				snake_case_ :					Tuple					,				snake_case_ :					List[Any]					,				snake_case_ :					Union[str, Any]=False   ):
    __a :  List[str]						=  super()._prepare_for_class(snake_case_					,				snake_case_					,				return_labels=snake_case_   )

    if return_labels:
     if model_class in get_values(snake_case_   ):
      __a :  Optional[int]						=  torch.zeros(
          (self.model_tester.batch_size, self.model_tester.seq_length)					,				dtype=torch.long					,				device=snake_case_   )
      __a :  int						=  torch.zeros(
          self.model_tester.batch_size					,				dtype=torch.long					,				device=snake_case_   )
    return inputs_dict
   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  List[Any]						=  MobileBertModelTester(self   )
    __a :  int						=  ConfigTester(self					,				config_class=snake_case_					,				hidden_size=3_7   )
   def       lowerCAmelCase			(self :					Union[str, Any]   ):
    self.config_tester.run_common_tests()
   def       lowerCAmelCase			(self :					Optional[Any]   ):
    __a :  Dict						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_model(*snake_case_   )
   def       lowerCAmelCase			(self :					str   ):
    __a :  Any						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_masked_lm(*snake_case_   )
   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  str						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_multiple_choice(*snake_case_   )
   def       lowerCAmelCase			(self :					Dict   ):
    __a :  Optional[Any]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*snake_case_   )
   def       lowerCAmelCase			(self :					int   ):
    __a :  Tuple						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_pretraining(*snake_case_   )
   def       lowerCAmelCase			(self :					List[Any]   ):
    __a :  int						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_question_answering(*snake_case_   )
   def       lowerCAmelCase			(self :					int   ):
    __a :  Optional[int]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_sequence_classification(*snake_case_   )







   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  Optional[int]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_mobilebert_for_token_classification(*snake_case_   )





def    __UpperCamelCase     (				lowerCAmelCase__							:				str  ):
 return torch.tensor(
     lowerCAmelCase__  ,   dtype=torch.long  ,   device=lowerCAmelCase__  ,   )


lowercase__       =1e-3



@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__  (       unittest.TestCase    ):
   @slow
   def       lowerCAmelCase			(self :					Any   ):
    __a :  Dict						=  MobileBertModel.from_pretrained('''google/mobilebert-uncased'''   ).to(snake_case_   )
    __a :  Tuple						=  _long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]   )
    with torch.no_grad():
     __a :  str						=  model(snake_case_   )[0]
    __a :  List[Any]						=  torch.Size((1, 9, 5_1_2)   )
    self.assertEqual(output.shape					,				snake_case_   )
    __a :  Union[str, Any]						=  torch.tensor(
        [
            [
                [-2.473_6526E07, 8.269_1656E04, 1.652_1838E05],
                [-5.754_1704E-01, 3.905_6022E00, 4.401_1507E00],
                [2.604_7359E00, 1.567_7652E00, -1.732_4188E-01],
            ]
        ]					,				device=snake_case_					,				)

    # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
    # ~1 difference, it's therefore not a good idea to measure using addition.
    # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
    # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
    __a :  List[str]						=  torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE   )
    __a :  Any						=  torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE   )

    self.assertTrue(lower_bound and upper_bound   )
