

import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a		=get_tests_dir("""fixtures/spiece.model""")






@require_sentencepiece
@require_tokenizers
class        A_		(       SCREAMING_SNAKE_CASE			,							unittest.TestCase				):
							_UpperCAmelCase    :  int						      =      AlbertTokenizer
							_UpperCAmelCase    :  Any						      =      AlbertTokenizerFast
							_UpperCAmelCase    :  Optional[Any]						      =      True
							_UpperCAmelCase    :  Union[str, Any]						      =      True
							_UpperCAmelCase    :  List[Any]						      =      True
							def     lowerCAmelCase     (		self		:			Optional[int]):
								super().setUp()

								# We have a SentencePiece fixture for testing
								__lowerCamelCase				:				Tuple					       = AlbertTokenizer(SCREAMING_SNAKE_CASE__)
								tokenizer.save_pretrained(self.tmpdirname)
							def     lowerCAmelCase     (		self		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]):
								__lowerCamelCase				:				Union[str, Any]					       = 'this is a test'
								__lowerCamelCase				:				Optional[Any]					       = 'this is a test'
								return input_text, output_text
							def     lowerCAmelCase     (		self		:			Dict):
								__lowerCamelCase				:				Optional[Any]					       = '<pad>'
								__lowerCamelCase				:				Any					       = 0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE__)   ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE__)   ,SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			str):
								__lowerCamelCase				:				List[str]					       = list(self.get_tokenizer().get_vocab().keys())

								self.assertEqual(vocab_keys[0]   ,'<pad>')
								self.assertEqual(vocab_keys[1]   ,'<unk>')
								self.assertEqual(vocab_keys[-1]   ,'▁eloquent')
								self.assertEqual(len(SCREAMING_SNAKE_CASE__)   ,3_0_0_0_0)
							def     lowerCAmelCase     (		self		:			Tuple):
								self.assertEqual(self.get_tokenizer().vocab_size   ,3_0_0_0_0)
							def     lowerCAmelCase     (		self		:			List[str]):
								if not self.test_rust_tokenizer:
									return

								__lowerCamelCase				:				Any					       = self.get_tokenizer()
								__lowerCamelCase				:				Dict					       = self.get_rust_tokenizer()

								__lowerCamelCase				:				int					       = 'I was born in 92000, and this is falsé.'

								__lowerCamelCase				:				List[str]					       = tokenizer.tokenize(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				Dict					       = rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__)
								self.assertListEqual(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__)

								__lowerCamelCase				:				Union[str, Any]					       = tokenizer.encode(SCREAMING_SNAKE_CASE__   ,add_special_tokens=SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				Dict					       = rust_tokenizer.encode(SCREAMING_SNAKE_CASE__   ,add_special_tokens=SCREAMING_SNAKE_CASE__)
								self.assertListEqual(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__)

								__lowerCamelCase				:				Tuple					       = self.get_rust_tokenizer()
								__lowerCamelCase				:				Optional[int]					       = tokenizer.encode(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				int					       = rust_tokenizer.encode(SCREAMING_SNAKE_CASE__)
								self.assertListEqual(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			Dict):
								__lowerCamelCase				:				List[Any]					       = AlbertTokenizer(SCREAMING_SNAKE_CASE__   ,keep_accents=SCREAMING_SNAKE_CASE__)

								__lowerCamelCase				:				Optional[int]					       = tokenizer.tokenize('This is a test')
								self.assertListEqual(SCREAMING_SNAKE_CASE__   ,['▁this', '▁is', '▁a', '▁test'])

								self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__)   ,[4_8, 2_5, 2_1, 1_2_8_9])

								__lowerCamelCase				:				str					       = tokenizer.tokenize('I was born in 92000, and this is falsé.')
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE__   ,['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.'])
								__lowerCamelCase				:				Any					       = tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__)
								self.assertListEqual(SCREAMING_SNAKE_CASE__   ,[3_1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9])

								__lowerCamelCase				:				Dict					       = tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__)
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE__   ,['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']   ,)
							def     lowerCAmelCase     (		self		:			Optional[Any]):
								__lowerCamelCase				:				Optional[Any]					       = AlbertTokenizer(SCREAMING_SNAKE_CASE__)

								__lowerCamelCase				:				str					       = tokenizer.encode('sequence builders')
								__lowerCamelCase				:				Tuple					       = tokenizer.encode('multi-sequence build')

								__lowerCamelCase				:				Union[str, Any]					       = tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				Optional[int]					       = tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__)

								assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
								assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
								    tokenizer.sep_token_id
								]


							@slow
							def     lowerCAmelCase     (		self		:			List[str]):
								# fmt: off
								__lowerCamelCase				:				Any					       = {'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_1_9_7_0, 1_3, 5, 6_0_9_2, 1_6_7, 2_8, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 7_0_2_8, 1_2_0_5_1, 1_8, 1_7, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 3_5_1_5, 1_8_6_8_4, 8, 4_4_6_1, 6, 1_9_2_7, 2_9_7, 8, 1_2_0_6_0, 2_6_0_7, 1_8, 1_3, 5, 4_4_6_1, 1_5, 1_0_5_3_8, 3_8, 8, 1_3_5, 1_5, 8_2_2, 5_8, 1_5, 9_9_3, 1_0_3_6_3, 1_5, 1_4_6_0, 8_0_0_5, 4_4_6_1, 1_5, 9_9_3, 2_5_5, 2_3_2_8, 9, 9, 9, 6, 2_6, 1_1_1_2, 8_1_6, 3_2_6_0, 1_3, 5, 1_0_3, 2_3_7_7, 6, 1_7, 1_1_1_2, 8_1_6, 2_7_8_2, 1_3, 5, 1_0_3, 1_0_6_4_1, 6, 2_9, 8_4, 2_5_1_2, 2_4_3_0, 7_8_2, 1_8_6_8_4, 2_7_6_1, 1_9, 8_0_8, 2_4_3_0, 2_5_5_6, 1_7, 8_5_5, 1_4_8_0, 9_4_7_7, 4_0_9_1, 1_2_8, 1_1_7_1_2, 1_5, 7_1_0_3, 2_1_5_3, 6_7_3, 1_7, 2_4_8_8_3, 9_9_9_0, 9, 3], [2, 1_1_5_0_2, 2_5, 1_0_0_6, 2_0, 7_8_2, 8, 1_1_8_0_9, 8_5_5, 1_7_3_2, 1_9_3_9_3, 1_8_6_6_7, 3_7, 3_6_7, 2_1_0_1_8, 6_9, 1_8_5_4, 3_4, 1_1_8_6_0, 1_9_1_2_4, 2_7, 1_5_6, 2_2_5, 1_7, 1_9_3, 4_1_4_1, 1_9, 6_5, 9_1_2_4, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 1_4, 2_2_3_1, 8_8_6, 2_3_8_5, 1_7_6_5_9, 8_4, 1_4, 1_6_7_9_2, 1_9_5_2, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=SCREAMING_SNAKE_CASE__   ,model_name='albert-base-v2'   ,revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'   ,)











import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MODEL_FOR_PRETRAINING_MAPPING,
			    MegatronBertForCausalLM,
			    MegatronBertForMaskedLM,
			    MegatronBertForMultipleChoice,
			    MegatronBertForNextSentencePrediction,
			    MegatronBertForPreTraining,
			    MegatronBertForQuestionAnswering,
			    MegatronBertForSequenceClassification,
			    MegatronBertForTokenClassification,
			    MegatronBertModel,
			)






class 					__A    :

			'''simple docstring'''







			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=1_3       ,  __lowerCAmelCase=7       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=9_9       ,  __lowerCAmelCase=6_4       ,  __lowerCAmelCase=3_2       ,  __lowerCAmelCase=5       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=3_7       ,  __lowerCAmelCase="gelu"       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=5_1_2       ,  __lowerCAmelCase=1_6       ,  __lowerCAmelCase=2       ,  __lowerCAmelCase=0.02       ,  __lowerCAmelCase=3       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=None       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		parent
				lowerCamelCase__								=		batch_size
				lowerCamelCase__								=		seq_length
				lowerCamelCase__								=		is_training
				lowerCamelCase__								=		use_input_mask
				lowerCamelCase__								=		use_token_type_ids
				lowerCamelCase__								=		use_labels
				lowerCamelCase__								=		vocab_size
				lowerCamelCase__								=		hidden_size
				lowerCamelCase__								=		embedding_size
				lowerCamelCase__								=		num_hidden_layers
				lowerCamelCase__								=		num_attention_heads
				lowerCamelCase__								=		intermediate_size
				lowerCamelCase__								=		hidden_act
				lowerCamelCase__								=		hidden_dropout_prob
				lowerCamelCase__								=		attention_probs_dropout_prob
				lowerCamelCase__								=		max_position_embeddings
				lowerCamelCase__								=		type_vocab_size
				lowerCamelCase__								=		type_sequence_label_size
				lowerCamelCase__								=		initializer_range
				lowerCamelCase__								=		num_labels
				lowerCamelCase__								=		num_choices
				lowerCamelCase__								=		scope





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size							)

				lowerCamelCase__								=		None
				if self.use_input_mask:
					lowerCamelCase__								=		random_attention_mask([self.batch_size, self.seq_length]							)

				lowerCamelCase__								=		None
				if self.use_token_type_ids:
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size							)

				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				if self.use_labels:
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.type_sequence_label_size							)
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels							)
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.num_choices							)

				lowerCamelCase__								=		self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return MegatronBertConfig(
				    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  embedding_size=self.embedding_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  is_decoder=__lowerCAmelCase       ,  initializer_range=self.initializer_range       ,  )





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertModel(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase							)
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase							)
				lowerCamelCase__								=		model(__lowerCAmelCase							)

				self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)							)
				self.parent.assertEqual(result.pooler_output.shape       ,  (self.batch_size, self.hidden_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForMaskedLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForCausalLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForNextSentencePrediction(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, 2)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForPreTraining(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase       ,  next_sentence_label=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.prediction_logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)
				self.parent.assertEqual(result.seq_relationship_logits.shape       ,  (self.batch_size, 2)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForQuestionAnswering(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  start_positions=__lowerCAmelCase       ,  end_positions=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.start_logits.shape       ,  (self.batch_size, self.seq_length)							)
				self.parent.assertEqual(result.end_logits.shape       ,  (self.batch_size, self.seq_length)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_labels
				lowerCamelCase__								=		MegatronBertForSequenceClassification(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_labels)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_labels
				lowerCamelCase__								=		MegatronBertForTokenClassification(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.num_labels)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_choices
				lowerCamelCase__								=		MegatronBertForMultipleChoice(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		input_ids.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
				lowerCamelCase__								=		token_type_ids.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
				lowerCamelCase__								=		input_mask.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_choices)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.prepare_config_and_inputs()
				(
				    (
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	
				)								=		config_and_inputs
				lowerCamelCase__								=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
				return config, inputs_dict






@require_torch
class 					__A    (    lowerCAmelCase			,				lowerCAmelCase			,				unittest.TestCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						(
			    (
			        MegatronBertModel,
			        MegatronBertForMaskedLM,
			        MegatronBertForCausalLM,
			        MegatronBertForMultipleChoice,
			        MegatronBertForNextSentencePrediction,
			        MegatronBertForPreTraining,
			        MegatronBertForQuestionAnswering,
			        MegatronBertForSequenceClassification,
			        MegatronBertForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCAmelCase_							     =						(
			    {
			        """feature-extraction""": MegatronBertModel,
			        """fill-mask""": MegatronBertForMaskedLM,
			        """question-answering""": MegatronBertForQuestionAnswering,
			        """text-classification""": MegatronBertForSequenceClassification,
			        """text-generation""": MegatronBertForCausalLM,
			        """token-classification""": MegatronBertForTokenClassification,
			        """zero-shot""": MegatronBertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCAmelCase_							     =						True
			# test_resize_embeddings = False
			lowerCAmelCase_							     =						False





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase=False							):



				'''simple docstring'''







				lowerCamelCase__								=		super()._prepare_for_class(__lowerCAmelCase       ,  __lowerCAmelCase       ,  return_labels=__lowerCAmelCase							)

				if return_labels:
					if model_class in get_values(__lowerCAmelCase							):
						lowerCamelCase__								=		torch.zeros(
						    (self.model_tester.batch_size, self.model_tester.seq_length)       ,  dtype=torch.long       ,  device=__lowerCAmelCase							)
						lowerCamelCase__								=		torch.zeros(
						    self.model_tester.batch_size       ,  dtype=torch.long       ,  device=__lowerCAmelCase							)
				return inputs_dict





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertModelTester(self							)
				lowerCamelCase__								=		ConfigTester(self       ,  config_class=__lowerCAmelCase       ,  hidden_size=3_7							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.config_tester.run_common_tests()





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_model(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_pretraining(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_question_answering(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_token_classification(*__lowerCAmelCase							)





def lowerCAmelCase__(__snake_case							)				->      List[str]:







	'''simple docstring'''







	return torch.tensor(
	    __snake_case       ,dtype=torch.long       ,device=__snake_case       ,)


_a 						=      1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class 					__A    (    unittest.TestCase ):

			'''simple docstring'''







			@slow
			@unittest.skip('''Model is not available.'''							)
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		'''nvidia/megatron-bert-uncased-345m'''
				if "MYDIR" in os.environ:
					lowerCamelCase__								=		os.path.join(os.environ['''MYDIR''']       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		MegatronBertModel.from_pretrained(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.half()
				lowerCamelCase__								=		_long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]							)
				with torch.no_grad():
					lowerCamelCase__								=		model(__lowerCAmelCase							)[0]
				lowerCamelCase__								=		torch.Size((1, 9, 1_0_2_4)							)
				self.assertEqual(output.shape       ,  __lowerCAmelCase							)

				lowerCamelCase__								=		[-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
				for ii in range(3							):
					for jj in range(3							):
						lowerCamelCase__								=		output[0, ii, jj]
						lowerCamelCase__								=		expected[3 * ii + jj]
						lowerCamelCase__								=		'''ii={} jj={} a={} b={}'''.format(__lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							)
						self.assertTrue(math.isclose(__lowerCAmelCase       ,  __lowerCAmelCase       ,  rel_tol=__lowerCAmelCase       ,  abs_tol=__lowerCAmelCase							)       ,  msg=__lowerCAmelCase							)



