code: 
  from __future__ import annotations
  
  import unittest
  
  from transformers import AlbertConfig, is_tf_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_tf, slow
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
  		import tensorflow as tf
  
  		from transformers import TF_MODEL_FOR_PRETRAINING_MAPPING
  		from transformers.models.albert.modeling_tf_albert import (
  		    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
  		    TFAlbertForMaskedLM,
  		    TFAlbertForMultipleChoice,
  		    TFAlbertForPreTraining,
  		    TFAlbertForQuestionAnswering,
  		    TFAlbertForSequenceClassification,
  		    TFAlbertForTokenClassification,
  		    TFAlbertModel,
  		)
  
  class 						snake_case			:
  			'''simple docstring'''
  
  			def __init__(	self     :     str							,							lowerCAmelCase_     :     List[str]							,							lowerCAmelCase_     :     int=13							,							lowerCAmelCase_     :     Optional[int]=7							,							lowerCAmelCase_     :     Optional[Any]=True							,							lowerCAmelCase_     :     int=True							,							lowerCAmelCase_     :     Tuple=True							,							lowerCAmelCase_     :     Tuple=True							,							lowerCAmelCase_     :     List[Any]=99							,							lowerCAmelCase_     :     Union[str, Any]=16							,							lowerCAmelCase_     :     Optional[Any]=32							,							lowerCAmelCase_     :     Any=2							,							lowerCAmelCase_     :     Tuple=4							,							lowerCAmelCase_     :     Optional[Any]=37							,							lowerCAmelCase_     :     str="gelu"							,							lowerCAmelCase_     :     int=0.1							,							lowerCAmelCase_     :     Union[str, Any]=0.1							,							lowerCAmelCase_     :     Optional[int]=512							,							lowerCAmelCase_     :     Dict=16							,							lowerCAmelCase_     :     List[Any]=2							,							lowerCAmelCase_     :     List[Any]=0.02							,							lowerCAmelCase_     :     Tuple=3							,							lowerCAmelCase_     :     Any=4							,							lowerCAmelCase_     :     str=None							,							)					->					Optional[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       parent
  								SCREAMING_SNAKE_CASE_	      =       13
  								SCREAMING_SNAKE_CASE_	      =       7
  								SCREAMING_SNAKE_CASE_	      =       True
  								SCREAMING_SNAKE_CASE_	      =       True
  								SCREAMING_SNAKE_CASE_	      =       True
  								SCREAMING_SNAKE_CASE_	      =       True
  								SCREAMING_SNAKE_CASE_	      =       99
  								SCREAMING_SNAKE_CASE_	      =       16
  								SCREAMING_SNAKE_CASE_	      =       32
  								SCREAMING_SNAKE_CASE_	      =       2
  								SCREAMING_SNAKE_CASE_	      =       4
  								SCREAMING_SNAKE_CASE_	      =       37
  								SCREAMING_SNAKE_CASE_	      =       '''gelu'''
  								SCREAMING_SNAKE_CASE_	      =       0.1
  								SCREAMING_SNAKE_CASE_	      =       0.1
  								SCREAMING_SNAKE_CASE_	      =       512
  								SCREAMING_SNAKE_CASE_	      =       16
  								SCREAMING_SNAKE_CASE_	      =       2
  								SCREAMING_SNAKE_CASE_	      =       0.02
  								SCREAMING_SNAKE_CASE_	      =       3
  								SCREAMING_SNAKE_CASE_	      =       4
  								SCREAMING_SNAKE_CASE_	      =       None
  
  			def _lowercase (	self     :     int					)					->					Union[str, Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size					)
  
  								SCREAMING_SNAKE_CASE_	      =       None
  								if self.use_input_mask:
  													SCREAMING_SNAKE_CASE_	      =       random_attention_mask([self.batch_size, self.seq_length]					)
  
  								SCREAMING_SNAKE_CASE_	      =       None
  								if self.use_token_type_ids:
  													SCREAMING_SNAKE_CASE_	      =       ids_tensor([self.batch_size, self.seq_length]							,							self.type_vocab_size					)
  
  								SCREAMING_SNAKE_CASE_	      =       None
  								SCREAMING_SNAKE_CASE_	      =       None
  								SCREAMING_SNAKE_CASE_	      =       None
  								if self.use_labels:
  													SCREAMING_SNAKE_CASE_	      =       ids_tensor([self.batch_size]							,							self.type_sequence_label_size					)
  													SCREAMING_SNAKE_CASE_	      =       ids_tensor([self.batch_size, self.seq_length]							,							self.num_labels					)
  													SCREAMING_SNAKE_CASE_	      =       ids_tensor([self.batch_size]							,							self.num_choices					)
  
  								SCREAMING_SNAKE_CASE_	      =       AlbertConfig(
  								    vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							embedding_size=self.embedding_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							hidden_act=self.hidden_act							,							hidden_dropout_prob=self.hidden_dropout_prob							,							attention_probs_dropout_prob=self.attention_probs_dropout_prob							,							max_position_embeddings=self.max_position_embeddings							,							type_vocab_size=self.type_vocab_size							,							initializer_range=self.initializer_range							,							)
  
  								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  			def _lowercase (	self     :     int							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     Optional[Any]							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     int					)					->					int:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertModel(config=lowerCAmelCase_					)
  								# inputs = {'input_ids': input_ids,
  								#           'attention_mask': input_mask,
  								#           'token_type_ids': token_type_ids}
  								# sequence_output, pooled_output = model(**inputs)
  								SCREAMING_SNAKE_CASE_	      =       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  
  								SCREAMING_SNAKE_CASE_	      =       [input_ids, input_mask]
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  
  								self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size)					)
  								self.parent.assertEqual(result.pooler_output.shape							,							(self.batch_size, self.hidden_size)					)
  
  			def _lowercase (	self     :     List[Any]							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     str							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     Any							,							lowerCAmelCase_     :     Optional[Any]							,							lowerCAmelCase_     :     List[str]					)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.num_labels
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForPreTraining(config=lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  								self.parent.assertEqual(result.prediction_logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)					)
  								self.parent.assertEqual(result.sop_logits.shape							,							(self.batch_size, self.num_labels)					)
  
  			def _lowercase (	self     :     List[Any]							,							lowerCAmelCase_     :     List[str]							,							lowerCAmelCase_     :     Any							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     Any							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     Tuple					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForMaskedLM(config=lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  								self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)					)
  
  			def _lowercase (	self     :     List[str]							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     Union[str, Any]					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.num_labels
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForSequenceClassification(config=lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  								self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_labels)					)
  
  			def _lowercase (	self     :     Any							,							lowerCAmelCase_     :     Union[str, Any]							,							lowerCAmelCase_     :     List[str]							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     Optional[Any]							,							lowerCAmelCase_     :     int					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForQuestionAnswering(config=lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  								self.parent.assertEqual(result.start_logits.shape							,							(self.batch_size, self.seq_length)					)
  								self.parent.assertEqual(result.end_logits.shape							,							(self.batch_size, self.seq_length)					)
  
  			def _lowercase (	self     :     Union[str, Any]							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     str							,							lowerCAmelCase_     :     List[Any]							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     Optional[Any]							,							lowerCAmelCase_     :     str							,							lowerCAmelCase_     :     str					)					->					Any:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.num_choices
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForMultipleChoice(config=lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       tf.tile(tf.expand_dims(lowerCAmelCase_							,							1					)							,							(1, self.num_choices, 1)					)
  								SCREAMING_SNAKE_CASE_	      =       tf.tile(tf.expand_dims(lowerCAmelCase_							,							1					)							,							(1, self.num_choices, 1)					)
  								SCREAMING_SNAKE_CASE_	      =       tf.tile(tf.expand_dims(lowerCAmelCase_							,							1					)							,							(1, self.num_choices, 1)					)
  								SCREAMING_SNAKE_CASE_	      =       {
  								    '''input_ids''': multiple_choice_inputs_ids,
  								    '''attention_mask''': multiple_choice_input_mask,
  								    '''token_type_ids''': multiple_choice_token_type_ids,
  								}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  								self.parent.assertListEqual(list(result['''logits'''].shape					)							,							[self.batch_size, self.num_choices]					)
  
  			def _lowercase (	self     :     Any							,							lowerCAmelCase_     :     Any							,							lowerCAmelCase_     :     int							,							lowerCAmelCase_     :     Any							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     List[str]							,							lowerCAmelCase_     :     Optional[Any]					)					->					List[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.num_labels
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForTokenClassification(config=lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       {
  								    '''input_ids''': input_ids,
  								    '''attention_mask''': input_mask,
  								    '''token_type_ids''': token_type_ids,
  								}
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)
  								self.parent.assertListEqual(list(result['''logits'''].shape					)							,							[self.batch_size, self.seq_length, self.num_labels]					)
  
  
  
  
  			def _lowercase (	self     :     Any					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.prepare_config_and_inputs()
  								(
  								    (
  								    SCREAMING_SNAKE_CASE_
  								)      ,(
  								    SCREAMING_SNAKE_CASE_
  								)      ,(
  								    SCREAMING_SNAKE_CASE_
  								)      ,(
  								    SCREAMING_SNAKE_CASE_
  								)      ,(
  								    SCREAMING_SNAKE_CASE_
  								)      ,(
  								    SCREAMING_SNAKE_CASE_
  								)      ,(
  								    SCREAMING_SNAKE_CASE_
  								)      ,
  								)	      =       config_and_inputs
  								SCREAMING_SNAKE_CASE_	      =       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
  								return config, inputs_dict
  
  @require_tf
  class 						snake_case			(       lowerCAmelCase__	, lowerCAmelCase__	, unittest.TestCase	):
  			'''simple docstring'''
  
  			UpperCAmelCase       :       Dict       = (
  			    (
  			        TFAlbertModel,
  			        TFAlbertForPreTraining,
  			        TFAlbertForMaskedLM,
  			        TFAlbertForSequenceClassification,
  			        TFAlbertForQuestionAnswering,
  			        TFAlbertForTokenClassification,
  			        TFAlbertForMultipleChoice,
  			    )
  			    if is_tf_available()
  			    else ()
  			)
  			UpperCAmelCase       :       Optional[Any]       = (
  			    {
  			        """feature-extraction""": TFAlbertModel,
  			        """fill-mask""": TFAlbertForMaskedLM,
  			        """question-answering""": TFAlbertForQuestionAnswering,
  			        """text-classification""": TFAlbertForSequenceClassification,
  			        """token-classification""": TFAlbertForTokenClassification,
  			        """zero-shot""": TFAlbertForSequenceClassification,
  			    }
  			    if is_tf_available()
  			    else {}
  			)
  			UpperCAmelCase       :       Optional[int]       = False
  			UpperCAmelCase       :       Optional[Any]       = False
  
  			def _lowercase (	self     :     Dict							,							lowerCAmelCase_     :     Dict							,							lowerCAmelCase_     :     Optional[int]							,							lowerCAmelCase_     :     str=False					)					->					int:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       super()._prepare_for_class(lowerCAmelCase_							,							lowerCAmelCase_							,							return_labels=lowerCAmelCase_					)
  
  								if return_labels:
  													if model_class in get_values(lowerCAmelCase_					):
  																		SCREAMING_SNAKE_CASE_	      =       tf.zeros(self.model_tester.batch_size							,							dtype=tf.intaa					)
  
  								return inputs_dict
  
  			def _lowercase (	self     :     Tuple					)					->					Optional[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertModelTester(self					)
  								SCREAMING_SNAKE_CASE_	      =       ConfigTester(self							,							config_class=lowerCAmelCase_							,							hidden_size=37					)
  
  			def _lowercase (	self     :     str					)					->					Dict:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								self.config_tester.run_common_tests()
  
  			def _lowercase (	self     :     Optional[Any]					)					->					Tuple:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_albert_model(*lowerCAmelCase_					)
  
  			def _lowercase (	self     :     str					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_albert_for_pretraining(*lowerCAmelCase_					)
  
  			def _lowercase (	self     :     int					)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_albert_for_masked_lm(*lowerCAmelCase_					)
  
  			def _lowercase (	self     :     str					)					->					Tuple:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_albert_for_multiple_choice(*lowerCAmelCase_					)
  
  			def _lowercase (	self     :     Optional[Any]					)					->					Dict:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_albert_for_sequence_classification(*lowerCAmelCase_					)
  
  			def _lowercase (	self     :     int					)					->					Union[str, Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.model_tester.prepare_config_and_inputs()
  								self.model_tester.create_and_check_albert_for_question_answering(*lowerCAmelCase_					)
  
  
  
  
  			@slow
  			def _lowercase (	self     :     Tuple					)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  													SCREAMING_SNAKE_CASE_	      =       TFAlbertModel.from_pretrained(lowerCAmelCase_					)
  													self.assertIsNotNone(lowerCAmelCase_					)
  
  @require_tf
  class 						snake_case			(       unittest.TestCase	):
  			'''simple docstring'''
  
  			@slow
  			def _lowercase (	self     :     Dict					)					->					Union[str, Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       TFAlbertForPreTraining.from_pretrained('''albert-base-v2'''					)
  								SCREAMING_SNAKE_CASE_	      =       tf.constant([[0, 1, 2, 3, 4, 5]]					)
  								SCREAMING_SNAKE_CASE_	      =       model(lowerCAmelCase_					)[0]
  
  								SCREAMING_SNAKE_CASE_	      =       [1, 6, 30_000]
  								self.assertEqual(output.shape							,							lowerCAmelCase_					)
  
  								SCREAMING_SNAKE_CASE_	      =       tf.constant(
  								    [
  								        [
  								            [4.595_668, 0.74_462_754, -1.818_147],
  								            [4.5_954_347, 0.7_454_184, -1.8_188_258],
  								            [4.5_954_905, 0.7_448_235, -1.8_182_316],
  								        ]
  								    ]					)
  								tf.debugging.assert_near(output[:, :3, :3]							,							lowerCAmelCase_							,							atol=1e-4					)
  
  
  
style_context: 
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_sentencepiece_available,
      is_tf_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  A_														=      {
      "configuration_switch_transformers": [
          "SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP",
          "SwitchTransformersConfig",
          "SwitchTransformersOnnxConfig",
      ]
  }
  
  try:
  		if not is_torch_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		A_														=      [
  		    "SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST",
  		    "SwitchTransformersEncoderModel",
  		    "SwitchTransformersForConditionalGeneration",
  		    "SwitchTransformersModel",
  		    "SwitchTransformersPreTrainedModel",
  		    "SwitchTransformersTop1Router",
  		    "SwitchTransformersSparseMLP",
  		]
  
  
  if TYPE_CHECKING:
  		from .configuration_switch_transformers import (
  		    SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP,
  		    SwitchTransformersConfig,
  		    SwitchTransformersOnnxConfig,
  		)
  
  		try:
  				if not is_torch_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .modeling_switch_transformers import (
  				    SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST,
  				    SwitchTransformersEncoderModel,
  				    SwitchTransformersForConditionalGeneration,
  				    SwitchTransformersModel,
  				    SwitchTransformersPreTrainedModel,
  				    SwitchTransformersSparseMLP,
  				    SwitchTransformersTopaRouter,
  				)
  
  
  else:
  		import sys
  
  		A_														=      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
  
  
  