


import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MODEL_FOR_PRETRAINING_MAPPING,
						    AlbertForMaskedLM,
						    AlbertForMultipleChoice,
						    AlbertForPreTraining,
						    AlbertForQuestionAnswering,
						    AlbertForSequenceClassification,
						    AlbertForTokenClassification,
						    AlbertModel,
						)
						from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class        snake_case_   :
					def __init__(      self   :Dict			,__snake_case   :Union[str, Any]			,__snake_case   :Tuple=13			,__snake_case   :Tuple=7			,__snake_case   :Tuple=True			,__snake_case   :Any=True			,__snake_case   :int=True			,__snake_case   :List[str]=True			,__snake_case   :Union[str, Any]=99			,__snake_case   :List[Any]=16			,__snake_case   :Tuple=36			,__snake_case   :str=6			,__snake_case   :int=6			,__snake_case   :str=6			,__snake_case   :Dict=37			,__snake_case   :str="gelu"			,__snake_case   :int=0.1			,__snake_case   :str=0.1			,__snake_case   :Any=5_12			,__snake_case   :Optional[Any]=16			,__snake_case   :List[Any]=2			,__snake_case   :str=0.02			,__snake_case   :Optional[int]=3			,__snake_case   :Optional[int]=4			,__snake_case   :Optional[Any]=None			,)					->    str:
												a__             =     parent
												a__             =     batch_size
												a__             =     seq_length
												a__             =     is_training
												a__             =     use_input_mask
												a__             =     use_token_type_ids
												a__             =     use_labels
												a__             =     vocab_size
												a__             =     embedding_size
												a__             =     hidden_size
												a__             =     num_hidden_layers
												a__             =     num_hidden_groups
												a__             =     num_attention_heads
												a__             =     intermediate_size
												a__             =     hidden_act
												a__             =     hidden_dropout_prob
												a__             =     attention_probs_dropout_prob
												a__             =     max_position_embeddings
												a__             =     type_vocab_size
												a__             =     type_sequence_label_size
												a__             =     initializer_range
												a__             =     num_labels
												a__             =     num_choices
												a__             =     scope
					def  lowerCamelCase__(      self   :List[str]      )					->    List[Any]:
												a__             =     ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size      )

												a__             =     None
												if self.use_input_mask:
																			a__             =     random_attention_mask([self.batch_size, self.seq_length]      )

												a__             =     None
												if self.use_token_type_ids:
																			a__             =     ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size      )

												a__             =     None
												a__             =     None
												a__             =     None
												if self.use_labels:
																			a__             =     ids_tensor([self.batch_size]			,self.type_sequence_label_size      )
																			a__             =     ids_tensor([self.batch_size, self.seq_length]			,self.num_labels      )
																			a__             =     ids_tensor([self.batch_size]			,self.num_choices      )

												a__             =     self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    int:
												return AlbertConfig(
												    vocab_size=self.vocab_size			,hidden_size=self.hidden_size			,num_hidden_layers=self.num_hidden_layers			,num_attention_heads=self.num_attention_heads			,intermediate_size=self.intermediate_size			,hidden_act=self.hidden_act			,hidden_dropout_prob=self.hidden_dropout_prob			,attention_probs_dropout_prob=self.attention_probs_dropout_prob			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_range=self.initializer_range			,num_hidden_groups=self.num_hidden_groups			,)
					def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :Optional[int]			,__snake_case   :Union[str, Any]			,__snake_case   :Dict			,__snake_case   :Optional[Any]			,__snake_case   :Union[str, Any]			,__snake_case   :Union[str, Any]			,__snake_case   :List[str]      )					->    Dict:
												a__             =     AlbertModel(config=__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     model(__snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case      )
												a__             =     model(__snake_case			,token_type_ids=__snake_case      )
												a__             =     model(__snake_case      )
												self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.hidden_size)      )
												self.parent.assertEqual(result.pooler_output.shape			,(self.batch_size, self.hidden_size)      )
					def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :List[Any]			,__snake_case   :int			,__snake_case   :Tuple			,__snake_case   :Tuple			,__snake_case   :List[Any]			,__snake_case   :Dict			,__snake_case   :Optional[Any]      )					->    Optional[int]:
												a__             =     AlbertForPreTraining(config=__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     model(
												    __snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case			,labels=__snake_case			,sentence_order_label=__snake_case			,)
												self.parent.assertEqual(result.prediction_logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)      )
												self.parent.assertEqual(result.sop_logits.shape			,(self.batch_size, config.num_labels)      )
					def  lowerCamelCase__(      self   :Optional[Any]			,__snake_case   :List[Any]			,__snake_case   :Optional[int]			,__snake_case   :str			,__snake_case   :Any			,__snake_case   :Union[str, Any]			,__snake_case   :Any			,__snake_case   :str      )					->    str:
												a__             =     AlbertForMaskedLM(config=__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     model(__snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case			,labels=__snake_case      )
												self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)      )
					def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :int			,__snake_case   :List[str]			,__snake_case   :List[str]			,__snake_case   :Union[str, Any]			,__snake_case   :List[Any]			,__snake_case   :Union[str, Any]			,__snake_case   :List[Any]      )					->    Tuple:
												a__             =     AlbertForQuestionAnswering(config=__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     model(
												    __snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case			,start_positions=__snake_case			,end_positions=__snake_case			,)
												self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)      )
												self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)      )
					def  lowerCamelCase__(      self   :List[Any]			,__snake_case   :List[Any]			,__snake_case   :Any			,__snake_case   :Union[str, Any]			,__snake_case   :Dict			,__snake_case   :List[str]			,__snake_case   :List[str]			,__snake_case   :Tuple      )					->    Union[str, Any]:
												a__             =     self.num_labels
												a__             =     AlbertForSequenceClassification(__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     model(__snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case			,labels=__snake_case      )
												self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)      )
					def  lowerCamelCase__(      self   :Union[str, Any]			,__snake_case   :Dict			,__snake_case   :Any			,__snake_case   :List[Any]			,__snake_case   :Union[str, Any]			,__snake_case   :str			,__snake_case   :List[Any]			,__snake_case   :int      )					->    List[Any]:
												a__             =     self.num_labels
												a__             =     AlbertForTokenClassification(config=__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     model(__snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case			,labels=__snake_case      )
												self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.num_labels)      )
					def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :Any			,__snake_case   :List[Any]			,__snake_case   :Any			,__snake_case   :List[Any]			,__snake_case   :Optional[int]			,__snake_case   :int			,__snake_case   :int      )					->    Optional[Any]:
												a__             =     self.num_choices
												a__             =     AlbertForMultipleChoice(config=__snake_case      )
												model.to(__snake_case      )
												model.eval()
												a__             =     input_ids.unsqueeze(1      ).expand(-1			,self.num_choices			,-1      ).contiguous()
												a__             =     token_type_ids.unsqueeze(1      ).expand(-1			,self.num_choices			,-1      ).contiguous()
												a__             =     input_mask.unsqueeze(1      ).expand(-1			,self.num_choices			,-1      ).contiguous()
												a__             =     model(
												    __snake_case			,attention_mask=__snake_case			,token_type_ids=__snake_case			,labels=__snake_case			,)
												self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_choices)      )



					def  lowerCamelCase__(      self   :List[str]      )					->    Dict:
												a__             =     self.prepare_config_and_inputs()
												(
												    (
												    a__
												)      ,					(
												    a__
												)      ,					(
												    a__
												)      ,					(
												    a__
												)      ,					(
												    a__
												)      ,					(
												    a__
												)      ,					(
												    a__
												)      ,					
												)             =     config_and_inputs
												a__             =     {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
												return config, inputs_dict







@require_torch
class        snake_case_   (lowerCamelCase_	,						lowerCamelCase_	,						unittest.TestCase      ):
					UpperCAmelCase__					:    List[str]          =			(
					    (
					        AlbertModel,
					        AlbertForPreTraining,
					        AlbertForMaskedLM,
					        AlbertForMultipleChoice,
					        AlbertForSequenceClassification,
					        AlbertForTokenClassification,
					        AlbertForQuestionAnswering,
					    )
					    if is_torch_available()
					    else ()
					)
					UpperCAmelCase__					:    Optional[Any]          =			(
					    {
					        '''feature-extraction''': AlbertModel,
					        '''fill-mask''': AlbertForMaskedLM,
					        '''question-answering''': AlbertForQuestionAnswering,
					        '''text-classification''': AlbertForSequenceClassification,
					        '''token-classification''': AlbertForTokenClassification,
					        '''zero-shot''': AlbertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					UpperCAmelCase__					:    Optional[Any]          =			True
					def  lowerCamelCase__(      self   :Any			,__snake_case   :Any			,__snake_case   :Dict			,__snake_case   :List[Any]=False      )					->    str:
												a__             =     super()._prepare_for_class(__snake_case			,__snake_case			,return_labels=__snake_case      )

												if return_labels:
																			if model_class in get_values(__snake_case      ):
																										a__             =     torch.zeros(
																										    (self.model_tester.batch_size, self.model_tester.seq_length)			,dtype=torch.long			,device=__snake_case      )
																										a__             =     torch.zeros(
																										    self.model_tester.batch_size			,dtype=torch.long			,device=__snake_case      )
												return inputs_dict
					def  lowerCamelCase__(      self   :Dict      )					->    Tuple:
												a__             =     AlbertModelTester(self      )
												a__             =     ConfigTester(self			,config_class=__snake_case			,hidden_size=37      )
					def  lowerCamelCase__(      self   :Optional[int]      )					->    Optional[int]:
												self.config_tester.run_common_tests()
					def  lowerCamelCase__(      self   :str      )					->    Optional[Any]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*__snake_case      )
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    List[str]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_pretraining(*__snake_case      )
					def  lowerCamelCase__(      self   :List[Any]      )					->    List[Any]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*__snake_case      )
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    List[str]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*__snake_case      )
					def  lowerCamelCase__(      self   :List[str]      )					->    List[str]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*__snake_case      )
					def  lowerCamelCase__(      self   :int      )					->    Optional[int]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*__snake_case      )
					def  lowerCamelCase__(      self   :Optional[Any]      )					->    Union[str, Any]:
												a__             =     self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																			a__             =     type
																			self.model_tester.create_and_check_model(*__snake_case      )



					@slow
					def  lowerCamelCase__(      self   :List[str]      )					->    Union[str, Any]:
												for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			a__             =     AlbertModel.from_pretrained(__snake_case      )
																			self.assertIsNotNone(__snake_case      )







@require_torch
class        snake_case_   (unittest.TestCase      ):
					@slow
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    Any:
												a__             =     AlbertModel.from_pretrained('albert-base-v2'      )
												a__             =     torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]      )
												a__             =     torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
												with torch.no_grad():
																			a__             =     model(__snake_case			,attention_mask=__snake_case      )[0]
												a__             =     torch.Size((1, 11, 7_68)      )
												self.assertEqual(output.shape			,__snake_case      )
												a__             =     torch.tensor(
												    [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]      )

												self.assertTrue(torch.allclose(output[:, 1:4, 1:4]			,__snake_case			,atol=1E-4      )      )








import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
						from sagemaker import Session, TrainingJobAnalytics
						from sagemaker.huggingface import HuggingFace







@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''	,						'''False'''      )      ) is not True	,						reason='''Skipping test because should only be run when releasing minor transformers version'''	,						)
@pytest.mark.usefixtures('''sm_env'''      )
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 6_5_0, '''eval_accuracy''': 0.6, '''eval_loss''': 0.9},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 6_0_0, '''eval_accuracy''': 0.3, '''eval_loss''': 0.9},
        },
    ]      )
class        snake_case_   (unittest.TestCase      ):
					def  lowerCamelCase__(      self   :Dict      )					->    Union[str, Any]:
												if self.framework == "pytorch":
																			subprocess.run(
																			    F'cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'.split()			,encoding='utf-8'			,check=__snake_case			,)
												assert hasattr(self			,'env'      )
					def  lowerCamelCase__(      self   :List[Any]			,__snake_case   :Optional[Any]=1      )					->    int:
												# creates estimator
												return HuggingFace(
												    entry_point=self.script			,source_dir=self.env.test_path			,role=self.env.role			,image_uri=self.env.image_uri			,base_job_name=F'{self.env.base_job_name}-single'			,instance_count=__snake_case			,instance_type=self.instance_type			,debugger_hook_config=__snake_case			,hyperparameters={**self.env.hyperparameters, 'model_name_or_path': self.model_name_or_path}			,metric_definitions=self.env.metric_definitions			,py_version='py36'			,)
					def  lowerCamelCase__(      self   :Dict			,__snake_case   :str      )					->    Optional[int]:
												TrainingJobAnalytics(__snake_case      ).export_csv(F'{self.env.test_path}/{job_name}_metrics.csv'      )



					def  lowerCamelCase__(      self   :Tuple      )					->    Union[str, Any]:
												# create estimator
												a__             =     self.create_estimator()

												# run training
												estimator.fit()

												# result dataframe
												a__             =     TrainingJobAnalytics(estimator.latest_training_job.name      ).dataframe()

												# extract kpis
												a__             =     list(result_metrics_df[result_metrics_df.metric_name == 'eval_accuracy']['value']      )
												a__             =     list(result_metrics_df[result_metrics_df.metric_name == 'eval_loss']['value']      )
												# get train time from SageMaker job, this includes starting, preprocessing, stopping
												a__             =     (
												    Session().describe_training_job(estimator.latest_training_job.name      ).get('TrainingTimeInSeconds'			,99_99_99      )
												)

												# assert kpis
												assert train_runtime <= self.results["train_runtime"]
												assert all(t >= self.results['eval_accuracy'] for t in eval_accuracy      )
												assert all(t <= self.results['eval_loss'] for t in eval_loss      )

												# dump tests result into json file to share in PR
												with open(F'{estimator.latest_training_job.name}.json'			,'w'      ) as outfile:
																			json.dump({'train_time': train_runtime, 'eval_accuracy': eval_accuracy, 'eval_loss': eval_loss}			,__snake_case      )




