



import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    AlbertForMaskedLM,
				    AlbertForMultipleChoice,
				    AlbertForPreTraining,
				    AlbertForQuestionAnswering,
				    AlbertForSequenceClassification,
				    AlbertForTokenClassification,
				    AlbertModel,
				)
				from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class       a							:



		def __init__(     self     :int   ,__lowercase     :Optional[int]   ,__lowercase     :List[Any]=1_3   ,__lowercase     :Tuple=7   ,__lowercase     :List[Any]=True   ,__lowercase     :Union[str, Any]=True   ,__lowercase     :int=True   ,__lowercase     :List[Any]=True   ,__lowercase     :Tuple=9_9   ,__lowercase     :List[str]=1_6   ,__lowercase     :Optional[int]=3_6   ,__lowercase     :Optional[int]=6   ,__lowercase     :Optional[int]=6   ,__lowercase     :Tuple=6   ,__lowercase     :Any=3_7   ,__lowercase     :Dict="gelu"   ,__lowercase     :List[str]=0.1   ,__lowercase     :int=0.1   ,__lowercase     :Dict=5_1_2   ,__lowercase     :Tuple=1_6   ,__lowercase     :Tuple=2   ,__lowercase     :List[Any]=0.02   ,__lowercase     :Tuple=3   ,__lowercase     :Optional[Any]=4   ,__lowercase     :List[str]=None   ,):
						snake_case__	:      int     		=	parent
						snake_case__	:      List[Any]     		=	batch_size
						snake_case__	:      int     		=	seq_length
						snake_case__	:      str     		=	is_training
						snake_case__	:      Optional[int]     		=	use_input_mask
						snake_case__	:      Optional[Any]     		=	use_token_type_ids
						snake_case__	:      Tuple     		=	use_labels
						snake_case__	:      Dict     		=	vocab_size
						snake_case__	:      List[Any]     		=	embedding_size
						snake_case__	:      int     		=	hidden_size
						snake_case__	:      Union[str, Any]     		=	num_hidden_layers
						snake_case__	:      Union[str, Any]     		=	num_hidden_groups
						snake_case__	:      int     		=	num_attention_heads
						snake_case__	:      List[Any]     		=	intermediate_size
						snake_case__	:      List[str]     		=	hidden_act
						snake_case__	:      List[Any]     		=	hidden_dropout_prob
						snake_case__	:      Union[str, Any]     		=	attention_probs_dropout_prob
						snake_case__	:      Any     		=	max_position_embeddings
						snake_case__	:      List[str]     		=	type_vocab_size
						snake_case__	:      Dict     		=	type_sequence_label_size
						snake_case__	:      List[str]     		=	initializer_range
						snake_case__	:      Union[str, Any]     		=	num_labels
						snake_case__	:      Union[str, Any]     		=	num_choices
						snake_case__	:      Tuple     		=	scope



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Tuple     		=	ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size  )

						snake_case__	:      Optional[Any]     		=	None
						if self.use_input_mask:
										snake_case__	:      int     		=	random_attention_mask([self.batch_size, self.seq_length]  )

						snake_case__	:      str     		=	None
						if self.use_token_type_ids:
										snake_case__	:      str     		=	ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size  )

						snake_case__	:      Optional[int]     		=	None
						snake_case__	:      List[str]     		=	None
						snake_case__	:      Tuple     		=	None
						if self.use_labels:
										snake_case__	:      Tuple     		=	ids_tensor([self.batch_size]   ,self.type_sequence_label_size  )
										snake_case__	:      Optional[Any]     		=	ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels  )
										snake_case__	:      List[str]     		=	ids_tensor([self.batch_size]   ,self.num_choices  )

						snake_case__	:      Dict     		=	self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 	__lowerCamelCase			(     self     :List[str]  ):
						return AlbertConfig(
						    vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,initializer_range=self.initializer_range   ,num_hidden_groups=self.num_hidden_groups   ,)



		def 	__lowerCamelCase			(     self     :List[str]   ,__lowercase     :str   ,__lowercase     :Any   ,__lowercase     :Union[str, Any]   ,__lowercase     :List[str]   ,__lowercase     :int   ,__lowercase     :Tuple   ,__lowercase     :Any  ):
						snake_case__	:      Dict     		=	AlbertModel(config=__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      List[str]     		=	model(__lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase  )
						snake_case__	:      int     		=	model(__lowercase   ,token_type_ids=__lowercase  )
						snake_case__	:      Dict     		=	model(__lowercase  )
						self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)  )
						self.parent.assertEqual(result.pooler_output.shape   ,(self.batch_size, self.hidden_size)  )



		def 	__lowerCamelCase			(     self     :List[Any]   ,__lowercase     :List[Any]   ,__lowercase     :int   ,__lowercase     :Dict   ,__lowercase     :Optional[Any]   ,__lowercase     :str   ,__lowercase     :List[str]   ,__lowercase     :List[str]  ):
						snake_case__	:      Optional[int]     		=	AlbertForPreTraining(config=__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      List[Any]     		=	model(
						    __lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase   ,labels=__lowercase   ,sentence_order_label=__lowercase   ,)
						self.parent.assertEqual(result.prediction_logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)  )
						self.parent.assertEqual(result.sop_logits.shape   ,(self.batch_size, config.num_labels)  )



		def 	__lowerCamelCase			(     self     :List[Any]   ,__lowercase     :str   ,__lowercase     :Union[str, Any]   ,__lowercase     :Any   ,__lowercase     :List[str]   ,__lowercase     :Tuple   ,__lowercase     :Optional[Any]   ,__lowercase     :Any  ):
						snake_case__	:      Dict     		=	AlbertForMaskedLM(config=__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      Dict     		=	model(__lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase   ,labels=__lowercase  )
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)  )



		def 	__lowerCamelCase			(     self     :int   ,__lowercase     :Tuple   ,__lowercase     :Tuple   ,__lowercase     :Optional[int]   ,__lowercase     :Dict   ,__lowercase     :int   ,__lowercase     :List[str]   ,__lowercase     :List[Any]  ):
						snake_case__	:      List[str]     		=	AlbertForQuestionAnswering(config=__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      Optional[int]     		=	model(
						    __lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase   ,start_positions=__lowercase   ,end_positions=__lowercase   ,)
						self.parent.assertEqual(result.start_logits.shape   ,(self.batch_size, self.seq_length)  )
						self.parent.assertEqual(result.end_logits.shape   ,(self.batch_size, self.seq_length)  )



		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :List[Any]   ,__lowercase     :str   ,__lowercase     :str   ,__lowercase     :Optional[Any]   ,__lowercase     :int   ,__lowercase     :Dict   ,__lowercase     :Any  ):
						snake_case__	:      Optional[int]     		=	self.num_labels
						snake_case__	:      Optional[Any]     		=	AlbertForSequenceClassification(__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      Tuple     		=	model(__lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase   ,labels=__lowercase  )
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_labels)  )



		def 	__lowerCamelCase			(     self     :Tuple   ,__lowercase     :Tuple   ,__lowercase     :Any   ,__lowercase     :Optional[Any]   ,__lowercase     :Tuple   ,__lowercase     :Optional[Any]   ,__lowercase     :Any   ,__lowercase     :int  ):
						snake_case__	:      Any     		=	self.num_labels
						snake_case__	:      Any     		=	AlbertForTokenClassification(config=__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      Any     		=	model(__lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase   ,labels=__lowercase  )
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.num_labels)  )



		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :Optional[Any]   ,__lowercase     :List[Any]   ,__lowercase     :List[Any]   ,__lowercase     :Any   ,__lowercase     :Optional[int]   ,__lowercase     :List[Any]   ,__lowercase     :int  ):
						snake_case__	:      Any     		=	self.num_choices
						snake_case__	:      Tuple     		=	AlbertForMultipleChoice(config=__lowercase  )
						model.to(__lowercase  )
						model.eval()
						snake_case__	:      Dict     		=	input_ids.unsqueeze(1  ).expand(-1   ,self.num_choices   ,-1  ).contiguous()
						snake_case__	:      Tuple     		=	token_type_ids.unsqueeze(1  ).expand(-1   ,self.num_choices   ,-1  ).contiguous()
						snake_case__	:      Optional[Any]     		=	input_mask.unsqueeze(1  ).expand(-1   ,self.num_choices   ,-1  ).contiguous()
						snake_case__	:      Optional[int]     		=	model(
						    __lowercase   ,attention_mask=__lowercase   ,token_type_ids=__lowercase   ,labels=__lowercase   ,)
						self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_choices)  )






		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Optional[Any]     		=	self.prepare_config_and_inputs()
						(
						    (
						    snake_case__
						)		, (
						    snake_case__
						)		, (
						    snake_case__
						)		, (
						    snake_case__
						)		, (
						    snake_case__
						)		, (
						    snake_case__
						)		, (
						    snake_case__
						)		, 
						)	:      Dict     		=	config_and_inputs
						snake_case__	:      List[Any]     		=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
						return config, inputs_dict


@require_torch
class       a							(			__lowerCamelCase  ,							__lowerCamelCase  ,							unittest.TestCase				):
		__lowerCAmelCase	:    str											=					(
		    (
		        AlbertModel,
		        AlbertForPreTraining,
		        AlbertForMaskedLM,
		        AlbertForMultipleChoice,
		        AlbertForSequenceClassification,
		        AlbertForTokenClassification,
		        AlbertForQuestionAnswering,
		    )
		    if is_torch_available()
		    else ()
		)
		__lowerCAmelCase	:    Tuple											=					(
		    {
		        """feature-extraction""": AlbertModel,
		        """fill-mask""": AlbertForMaskedLM,
		        """question-answering""": AlbertForQuestionAnswering,
		        """text-classification""": AlbertForSequenceClassification,
		        """token-classification""": AlbertForTokenClassification,
		        """zero-shot""": AlbertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__lowerCAmelCase	:    Dict											=					True



		def 	__lowerCamelCase			(     self     :Tuple   ,__lowercase     :List[Any]   ,__lowercase     :Any   ,__lowercase     :int=False  ):
						snake_case__	:      str     		=	super()._prepare_for_class(__lowercase   ,__lowercase   ,return_labels=__lowercase  )

						if return_labels:
										if model_class in get_values(__lowercase  ):
														snake_case__	:      Dict     		=	torch.zeros(
														    (self.model_tester.batch_size, self.model_tester.seq_length)   ,dtype=torch.long   ,device=__lowercase  )
														snake_case__	:      List[Any]     		=	torch.zeros(
														    self.model_tester.batch_size   ,dtype=torch.long   ,device=__lowercase  )
						return inputs_dict



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      str     		=	AlbertModelTester(self  )
						snake_case__	:      List[str]     		=	ConfigTester(self   ,config_class=__lowercase   ,hidden_size=3_7  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						self.config_tester.run_common_tests()



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      List[str]     		=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__lowercase  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						snake_case__	:      Dict     		=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_pretraining(*__lowercase  )



		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      List[Any]     		=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*__lowercase  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						snake_case__	:      List[str]     		=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*__lowercase  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						snake_case__	:      List[Any]     		=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      List[str]     		=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      List[Any]     		=	self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
										snake_case__	:      int     		=	type
										self.model_tester.create_and_check_model(*__lowercase  )






		@slow
		def 	__lowerCamelCase			(     self     :int  ):
						for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										snake_case__	:      Optional[Any]     		=	AlbertModel.from_pretrained(__lowercase  )
										self.assertIsNotNone(__lowercase  )


@require_torch
class       a							(			unittest.TestCase				):



		@slow
		def 	__lowerCamelCase			(     self     :List[Any]  ):
						snake_case__	:      Tuple     		=	AlbertModel.from_pretrained('''albert-base-v2'''  )
						snake_case__	:      Tuple     		=	torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]  )
						snake_case__	:      List[Any]     		=	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]  )
						with torch.no_grad():
										snake_case__	:      Optional[Any]     		=	model(__lowercase   ,attention_mask=__lowercase  )[0]
						snake_case__	:      Union[str, Any]     		=	torch.Size((1, 1_1, 7_6_8)  )
						self.assertEqual(output.shape   ,__lowercase  )
						snake_case__	:      Optional[Any]     		=	torch.tensor(
						    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]  )

						self.assertTrue(torch.allclose(output[:, 1:4, 1:4]   ,__lowercase   ,atol=1e-4  )  )






from importlib import import_module

from .logging import get_logger


A__								=						get_logger(__name__)


class       a							:



		def __init__(     self     :Optional[int]   ,__lowercase     :List[str]   ,__lowercase     :Any=None  ):
						snake_case__	:      List[str]     		=	attrs or []
						if module is not None:
										for key in module.__dict__:
														if key in attrs or not key.startswith('''__'''  ):
																		setattr(self   ,__lowercase   ,getattr(__lowercase   ,__lowercase  )  )
						snake_case__	:      Optional[Any]     		=	module._original_module if isinstance(__lowercase   ,_PatchedModuleObj  ) else module


class       a							:
		__lowerCAmelCase	:    Any											=					[]



		def __init__(     self     :List[str]   ,__lowercase     :Optional[Any]   ,__lowercase     :str   ,__lowercase     :Dict   ,__lowercase     :Any=None  ):
						snake_case__	:      Dict     		=	obj
						snake_case__	:      Dict     		=	target
						snake_case__	:      List[str]     		=	new
						snake_case__	:      int     		=	target.split('''.'''  )[0]
						snake_case__	:      List[str]     		=	{}
						snake_case__	:      Any     		=	attrs or []



		def __enter__(     self     :Tuple  ):
						*snake_case__		, snake_case__	:      str     		=	self.target.split('''.'''  )

						# Patch modules:
						# it's used to patch attributes of submodules like "os.path.join";
						# in this case we need to patch "os" and "os.path"

						for i in range(len(__lowercase  )  ):
										try:
														snake_case__	:      Any     		=	import_module('''.'''.join(submodules[: i + 1]  )  )
										except ModuleNotFoundError:
														continue
										# We iterate over all the globals in self.obj in case we find "os" or "os.path"
										for attr in self.obj.__dir__():
														snake_case__	:      Optional[int]     		=	getattr(self.obj   ,__lowercase  )
														# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
														# This allows to patch renamed modules like "from os import path as ospath".
														if obj_attr is submodule or (
														    (isinstance(__lowercase   ,_PatchedModuleObj  ) and obj_attr._original_module is submodule)
														):
																		snake_case__	:      List[Any]     		=	obj_attr
																		# patch at top level
																		setattr(self.obj   ,__lowercase   ,_PatchedModuleObj(__lowercase   ,attrs=self.attrs  )  )
																		snake_case__	:      List[Any]     		=	getattr(self.obj   ,__lowercase  )
																		# construct lower levels patches
																		for key in submodules[i + 1 :]:
																						setattr(__lowercase   ,__lowercase   ,_PatchedModuleObj(getattr(__lowercase   ,__lowercase   ,__lowercase  )   ,attrs=self.attrs  )  )
																						snake_case__	:      List[Any]     		=	getattr(__lowercase   ,__lowercase  )
																		# finally set the target attribute
																		setattr(__lowercase   ,__lowercase   ,self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

						if submodules:  # if it's an attribute of a submodule like "os.path.join"
										try:
														snake_case__	:      int     		=	getattr(import_module('''.'''.join(__lowercase  )  )   ,__lowercase  )
										except (AttributeError, ModuleNotFoundError):
														return
										# We iterate over all the globals in self.obj in case we find "os.path.join"
										for attr in self.obj.__dir__():
														# We don't check for the name of the global, but rather if its value *is* "os.path.join".
														# This allows to patch renamed attributes like "from os.path import join as pjoin".
														if getattr(self.obj   ,__lowercase  ) is attr_value:
																		snake_case__	:      str     		=	getattr(self.obj   ,__lowercase  )
																		setattr(self.obj   ,__lowercase   ,self.new  )
						elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
										snake_case__	:      str     		=	globals()['''__builtins__'''][target_attr]
										setattr(self.obj   ,__lowercase   ,self.new  )
						else:
										raise RuntimeError(F"""Tried to patch attribute {target_attr} instead of a submodule."""  )



		def __exit__(     self     :Tuple   ,*__lowercase     :Optional[int]  ):
						for attr in list(self.original  ):
										setattr(self.obj   ,__lowercase   ,self.original.pop(__lowercase  )  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						self.__enter__()
						self._active_patches.append(self  )






		def 	__lowerCamelCase			(     self     :Dict  ):
						try:
										self._active_patches.remove(self  )
						except ValueError:
										# If the patch hasn't been started this will fail
										return None

						return self.__exit__()

