





import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_a 						=      "▁"

_a 						=      get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
class 					__A    (    lowerCAmelCase			,				unittest.TestCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						BertGenerationTokenizer
			lowerCAmelCase_							     =						False
			lowerCAmelCase_							     =						True





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				super().setUp()

				lowerCamelCase__								=		BertGenerationTokenizer(__lowerCAmelCase       ,  keep_accents=__lowerCAmelCase							)
				tokenizer.save_pretrained(self.tmpdirname							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		'''<s>'''
				lowerCamelCase__								=		1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase							)       ,  __lowerCAmelCase							)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase							)       ,  __lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		list(self.get_tokenizer().get_vocab().keys()							)

				self.assertEqual(vocab_keys[0]       ,  '''<unk>'''							)
				self.assertEqual(vocab_keys[1]       ,  '''<s>'''							)
				self.assertEqual(vocab_keys[-1]       ,  '''<pad>'''							)
				self.assertEqual(len(__lowerCAmelCase							)       ,  1_0_0_2							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.assertEqual(self.get_tokenizer().vocab_size       ,  1_0_0_0							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BertGenerationTokenizer(__lowerCAmelCase       ,  keep_accents=__lowerCAmelCase							)

				lowerCamelCase__								=		tokenizer.tokenize('''This is a test'''							)
				self.assertListEqual(__lowerCAmelCase       ,  ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(__lowerCAmelCase							)       ,  [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]       ,  )

				lowerCamelCase__								=		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
				self.assertListEqual(
				    __lowerCAmelCase       ,  [
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''9''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''é''',
				        '''.''',
				    ]       ,  )
				lowerCamelCase__								=		tokenizer.convert_tokens_to_ids(__lowerCAmelCase							)
				self.assertListEqual(
				    __lowerCAmelCase       ,  [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]       ,  )

				lowerCamelCase__								=		tokenizer.convert_ids_to_tokens(__lowerCAmelCase							)
				self.assertListEqual(
				    __lowerCAmelCase       ,  [
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''<unk>''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''<unk>''',
				        '''.''',
				    ]       ,  )





			@cached_property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return BertGenerationTokenizer.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''							)





			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		'''Hello World!'''
				lowerCamelCase__								=		[1_8_5_3_6, 2_2_6_0, 1_0_1]

				self.assertListEqual(__lowerCAmelCase       ,  self.big_tokenizer.encode(__lowerCAmelCase							)							)





			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		(
				    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
				    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
				)
				lowerCamelCase__								=		[
				    8_7_1,
				    4_1_9,
				    3_5_8,
				    9_4_6,
				    9_9_1,
				    2_5_2_1,
				    4_5_2,
				    3_5_8,
				    1_3_5_7,
				    3_8_7,
				    7_7_5_1,
				    3_5_3_6,
				    1_1_2,
				    9_8_5,
				    4_5_6,
				    1_2_6,
				    8_6_5,
				    9_3_8,
				    5_4_0_0,
				    5_7_3_4,
				    4_5_8,
				    1_3_6_8,
				    4_6_7,
				    7_8_6,
				    2_4_6_2,
				    5_2_4_6,
				    1_1_5_9,
				    6_3_3,
				    8_6_5,
				    4_5_1_9,
				    4_5_7,
				    5_8_2,
				    8_5_2,
				    2_5_5_7,
				    4_2_7,
				    9_1_6,
				    5_0_8,
				    4_0_5,
				    3_4_3_2_4,
				    4_9_7,
				    3_9_1,
				    4_0_8,
				    1_1_3_4_2,
				    1_2_4_4,
				    3_8_5,
				    1_0_0,
				    9_3_8,
				    9_8_5,
				    4_5_6,
				    5_7_4,
				    3_6_2,
				    1_2_5_9_7,
				    3_2_0_0,
				    3_1_2_9,
				    1_1_7_2,
				]

				self.assertListEqual(__lowerCAmelCase       ,  self.big_tokenizer.encode(__lowerCAmelCase							)							)





			@require_torch
			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				import torch

				from transformers import BertGenerationConfig, BertGenerationEncoder

				# Build sequence
				lowerCamelCase__								=		list(self.big_tokenizer.get_vocab().keys()							)[:1_0]
				lowerCamelCase__								=		''' '''.join(__lowerCAmelCase							)
				lowerCamelCase__								=		self.big_tokenizer.encode_plus(__lowerCAmelCase       ,  return_tensors='''pt'''       ,  return_token_type_ids=__lowerCAmelCase							)
				lowerCamelCase__								=		self.big_tokenizer.batch_encode_plus(
				    [sequence + ''' ''' + sequence]       ,  return_tensors='''pt'''       ,  return_token_type_ids=__lowerCAmelCase							)

				lowerCamelCase__								=		BertGenerationConfig()
				lowerCamelCase__								=		BertGenerationEncoder(__lowerCAmelCase							)

				assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

				with torch.no_grad():
					model(**__lowerCAmelCase							)
					model(**__lowerCAmelCase							)





			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		{'''input_ids''': [[3_9_2_8_6, 4_5_8, 3_6_3_3_5, 2_0_0_1, 4_5_6, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 7_7_4_6, 1_7_4_1, 1_1_1_5_7, 3_9_1, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 3_9_6_7, 3_5_4_1_2, 1_1_3, 4_9_3_6, 1_0_9, 3_8_7_0, 2_3_7_7, 1_1_3, 3_0_0_8_4, 4_5_7_2_0, 4_5_8, 1_3_4, 1_7_4_9_6, 1_1_2, 5_0_3, 1_1_6_7_2, 1_1_3, 1_1_8, 1_1_2, 5_6_6_5, 1_3_3_4_7, 3_8_6_8_7, 1_1_2, 1_4_9_6, 3_1_3_8_9, 1_1_2, 3_2_6_8, 4_7_2_6_4, 1_3_4, 9_6_2, 1_1_2, 1_6_3_7_7, 8_0_3_5, 2_3_1_3_0, 4_3_0, 1_2_1_6_9, 1_5_5_1_8, 2_8_5_9_2, 4_5_8, 1_4_6, 4_1_6_9_7, 1_0_9, 3_9_1, 1_2_1_6_9, 1_5_5_1_8, 1_6_6_8_9, 4_5_8, 1_4_6, 4_1_3_5_8, 1_0_9, 4_5_2, 7_2_6, 4_0_3_4, 1_1_1, 7_6_3, 3_5_4_1_2, 5_0_8_2, 3_8_8, 1_9_0_3, 1_1_1, 9_0_5_1, 3_9_1, 2_8_7_0, 4_8_9_1_8, 1_9_0_0, 1_1_2_3, 5_5_0, 9_9_8, 1_1_2, 9_5_8_6, 1_5_9_8_5, 4_5_5, 3_9_1, 4_1_0, 2_2_9_5_5, 3_7_6_3_6, 1_1_4], [4_4_8, 1_7_4_9_6, 4_1_9, 3_6_6_3, 3_8_5, 7_6_3, 1_1_3, 2_7_5_3_3, 2_8_7_0, 3_2_8_3, 1_3_0_4_3, 1_6_3_9, 2_4_7_1_3, 5_2_3, 6_5_6, 2_4_0_1_3, 1_8_5_5_0, 2_5_2_1, 5_1_7, 2_7_0_1_4, 2_1_2_4_4, 4_2_0, 1_2_1_2, 1_4_6_5, 3_9_1, 9_2_7, 4_8_3_3, 3_8_8, 5_7_8, 1_1_7_8_6, 1_1_4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4_8_4, 2_1_6_9, 7_6_8_7, 2_1_9_3_2, 1_8_1_4_6, 7_2_6, 3_6_3, 1_7_0_3_2, 3_3_9_1, 1_1_4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=__lowerCAmelCase       ,  model_name='''google/bert_for_seq_generation_L-24_bbc_encoder'''       ,  revision='''c817d1fd1be2ffa69431227a1fe320544943d4db'''       ,  )










import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MODEL_FOR_PRETRAINING_MAPPING,
			    MegatronBertForCausalLM,
			    MegatronBertForMaskedLM,
			    MegatronBertForMultipleChoice,
			    MegatronBertForNextSentencePrediction,
			    MegatronBertForPreTraining,
			    MegatronBertForQuestionAnswering,
			    MegatronBertForSequenceClassification,
			    MegatronBertForTokenClassification,
			    MegatronBertModel,
			)






class 					__A    :

			'''simple docstring'''







			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=1_3       ,  __lowerCAmelCase=7       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=9_9       ,  __lowerCAmelCase=6_4       ,  __lowerCAmelCase=3_2       ,  __lowerCAmelCase=5       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=3_7       ,  __lowerCAmelCase="gelu"       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=5_1_2       ,  __lowerCAmelCase=1_6       ,  __lowerCAmelCase=2       ,  __lowerCAmelCase=0.02       ,  __lowerCAmelCase=3       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=None       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		parent
				lowerCamelCase__								=		batch_size
				lowerCamelCase__								=		seq_length
				lowerCamelCase__								=		is_training
				lowerCamelCase__								=		use_input_mask
				lowerCamelCase__								=		use_token_type_ids
				lowerCamelCase__								=		use_labels
				lowerCamelCase__								=		vocab_size
				lowerCamelCase__								=		hidden_size
				lowerCamelCase__								=		embedding_size
				lowerCamelCase__								=		num_hidden_layers
				lowerCamelCase__								=		num_attention_heads
				lowerCamelCase__								=		intermediate_size
				lowerCamelCase__								=		hidden_act
				lowerCamelCase__								=		hidden_dropout_prob
				lowerCamelCase__								=		attention_probs_dropout_prob
				lowerCamelCase__								=		max_position_embeddings
				lowerCamelCase__								=		type_vocab_size
				lowerCamelCase__								=		type_sequence_label_size
				lowerCamelCase__								=		initializer_range
				lowerCamelCase__								=		num_labels
				lowerCamelCase__								=		num_choices
				lowerCamelCase__								=		scope





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size							)

				lowerCamelCase__								=		None
				if self.use_input_mask:
					lowerCamelCase__								=		random_attention_mask([self.batch_size, self.seq_length]							)

				lowerCamelCase__								=		None
				if self.use_token_type_ids:
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size							)

				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				if self.use_labels:
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.type_sequence_label_size							)
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels							)
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.num_choices							)

				lowerCamelCase__								=		self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return MegatronBertConfig(
				    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  embedding_size=self.embedding_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  is_decoder=__lowerCAmelCase       ,  initializer_range=self.initializer_range       ,  )





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertModel(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase							)
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase							)
				lowerCamelCase__								=		model(__lowerCAmelCase							)

				self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)							)
				self.parent.assertEqual(result.pooler_output.shape       ,  (self.batch_size, self.hidden_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForMaskedLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForCausalLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForNextSentencePrediction(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, 2)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForPreTraining(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase       ,  next_sentence_label=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.prediction_logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)
				self.parent.assertEqual(result.seq_relationship_logits.shape       ,  (self.batch_size, 2)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertForQuestionAnswering(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  start_positions=__lowerCAmelCase       ,  end_positions=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.start_logits.shape       ,  (self.batch_size, self.seq_length)							)
				self.parent.assertEqual(result.end_logits.shape       ,  (self.batch_size, self.seq_length)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_labels
				lowerCamelCase__								=		MegatronBertForSequenceClassification(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_labels)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_labels
				lowerCamelCase__								=		MegatronBertForTokenClassification(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.num_labels)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_choices
				lowerCamelCase__								=		MegatronBertForMultipleChoice(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		input_ids.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
				lowerCamelCase__								=		token_type_ids.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
				lowerCamelCase__								=		input_mask.unsqueeze(1							).expand(-1       ,  self.num_choices       ,  -1							).contiguous()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_choices)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.prepare_config_and_inputs()
				(
				    (
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	
				)								=		config_and_inputs
				lowerCamelCase__								=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
				return config, inputs_dict






@require_torch
class 					__A    (    lowerCAmelCase			,				lowerCAmelCase			,				unittest.TestCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						(
			    (
			        MegatronBertModel,
			        MegatronBertForMaskedLM,
			        MegatronBertForCausalLM,
			        MegatronBertForMultipleChoice,
			        MegatronBertForNextSentencePrediction,
			        MegatronBertForPreTraining,
			        MegatronBertForQuestionAnswering,
			        MegatronBertForSequenceClassification,
			        MegatronBertForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCAmelCase_							     =						(
			    {
			        """feature-extraction""": MegatronBertModel,
			        """fill-mask""": MegatronBertForMaskedLM,
			        """question-answering""": MegatronBertForQuestionAnswering,
			        """text-classification""": MegatronBertForSequenceClassification,
			        """text-generation""": MegatronBertForCausalLM,
			        """token-classification""": MegatronBertForTokenClassification,
			        """zero-shot""": MegatronBertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCAmelCase_							     =						True
			# test_resize_embeddings = False
			lowerCAmelCase_							     =						False





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase=False							):



				'''simple docstring'''







				lowerCamelCase__								=		super()._prepare_for_class(__lowerCAmelCase       ,  __lowerCAmelCase       ,  return_labels=__lowerCAmelCase							)

				if return_labels:
					if model_class in get_values(__lowerCAmelCase							):
						lowerCamelCase__								=		torch.zeros(
						    (self.model_tester.batch_size, self.model_tester.seq_length)       ,  dtype=torch.long       ,  device=__lowerCAmelCase							)
						lowerCamelCase__								=		torch.zeros(
						    self.model_tester.batch_size       ,  dtype=torch.long       ,  device=__lowerCAmelCase							)
				return inputs_dict





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		MegatronBertModelTester(self							)
				lowerCamelCase__								=		ConfigTester(self       ,  config_class=__lowerCAmelCase       ,  hidden_size=3_7							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.config_tester.run_common_tests()





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_model(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_pretraining(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_question_answering(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_megatron_bert_for_token_classification(*__lowerCAmelCase							)





def lowerCAmelCase__(__snake_case							)				->      List[str]:







	'''simple docstring'''







	return torch.tensor(
	    __snake_case       ,dtype=torch.long       ,device=__snake_case       ,)


_a 						=      1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class 					__A    (    unittest.TestCase ):

			'''simple docstring'''







			@slow
			@unittest.skip('''Model is not available.'''							)
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		'''nvidia/megatron-bert-uncased-345m'''
				if "MYDIR" in os.environ:
					lowerCamelCase__								=		os.path.join(os.environ['''MYDIR''']       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		MegatronBertModel.from_pretrained(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.half()
				lowerCamelCase__								=		_long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]							)
				with torch.no_grad():
					lowerCamelCase__								=		model(__lowerCAmelCase							)[0]
				lowerCamelCase__								=		torch.Size((1, 9, 1_0_2_4)							)
				self.assertEqual(output.shape       ,  __lowerCAmelCase							)

				lowerCamelCase__								=		[-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
				for ii in range(3							):
					for jj in range(3							):
						lowerCamelCase__								=		output[0, ii, jj]
						lowerCamelCase__								=		expected[3 * ii + jj]
						lowerCamelCase__								=		'''ii={} jj={} a={} b={}'''.format(__lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							)
						self.assertTrue(math.isclose(__lowerCAmelCase       ,  __lowerCAmelCase       ,  rel_tol=__lowerCAmelCase       ,  abs_tol=__lowerCAmelCase							)       ,  msg=__lowerCAmelCase							)



