






import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    MobileBertForMaskedLM,
		    MobileBertForMultipleChoice,
		    MobileBertForNextSentencePrediction,
		    MobileBertForPreTraining,
		    MobileBertForQuestionAnswering,
		    MobileBertForSequenceClassification,
		    MobileBertForTokenClassification,
		    MobileBertModel,
		)





class __lowerCamelCase :





	"""simple docstring"""






	def __init__(							self       ,    UpperCAmelCase       ,    UpperCAmelCase=13       ,    UpperCAmelCase=7       ,    UpperCAmelCase=True       ,    UpperCAmelCase=True       ,    UpperCAmelCase=True       ,    UpperCAmelCase=True       ,    UpperCAmelCase=99       ,    UpperCAmelCase=64       ,    UpperCAmelCase=32       ,    UpperCAmelCase=5       ,    UpperCAmelCase=4       ,    UpperCAmelCase=37       ,    UpperCAmelCase="gelu"       ,    UpperCAmelCase=0.1       ,    UpperCAmelCase=0.1       ,    UpperCAmelCase=512       ,    UpperCAmelCase=16       ,    UpperCAmelCase=2       ,    UpperCAmelCase=0.02       ,    UpperCAmelCase=3       ,    UpperCAmelCase=4       ,    UpperCAmelCase=None       ,    ):
							"""simple docstring"""







							_UpperCAmelCase									=			parent
							_UpperCAmelCase									=			batch_size
							_UpperCAmelCase									=			seq_length
							_UpperCAmelCase									=			is_training
							_UpperCAmelCase									=			use_input_mask
							_UpperCAmelCase									=			use_token_type_ids
							_UpperCAmelCase									=			use_labels
							_UpperCAmelCase									=			vocab_size
							_UpperCAmelCase									=			hidden_size
							_UpperCAmelCase									=			embedding_size
							_UpperCAmelCase									=			num_hidden_layers
							_UpperCAmelCase									=			num_attention_heads
							_UpperCAmelCase									=			intermediate_size
							_UpperCAmelCase									=			hidden_act
							_UpperCAmelCase									=			hidden_dropout_prob
							_UpperCAmelCase									=			attention_probs_dropout_prob
							_UpperCAmelCase									=			max_position_embeddings
							_UpperCAmelCase									=			type_vocab_size
							_UpperCAmelCase									=			type_sequence_label_size
							_UpperCAmelCase									=			initializer_range
							_UpperCAmelCase									=			num_labels
							_UpperCAmelCase									=			num_choices
							_UpperCAmelCase									=			scope

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]       ,    self.vocab_size    )

							_UpperCAmelCase									=			None
							if self.use_input_mask:
													_UpperCAmelCase									=			random_attention_mask([self.batch_size, self.seq_length]    )

							_UpperCAmelCase									=			None
							if self.use_token_type_ids:
													_UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]       ,    self.type_vocab_size    )

							_UpperCAmelCase									=			None
							_UpperCAmelCase									=			None
							_UpperCAmelCase									=			None
							if self.use_labels:
													_UpperCAmelCase									=			ids_tensor([self.batch_size]       ,    self.type_sequence_label_size    )
													_UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]       ,    self.num_labels    )
													_UpperCAmelCase									=			ids_tensor([self.batch_size]       ,    self.num_choices    )

							_UpperCAmelCase									=			self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							return MobileBertConfig(
							    vocab_size=self.vocab_size       ,    hidden_size=self.hidden_size       ,    num_hidden_layers=self.num_hidden_layers       ,    num_attention_heads=self.num_attention_heads       ,    intermediate_size=self.intermediate_size       ,    embedding_size=self.embedding_size       ,    hidden_act=self.hidden_act       ,    hidden_dropout_prob=self.hidden_dropout_prob       ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,    max_position_embeddings=self.max_position_embeddings       ,    type_vocab_size=self.type_vocab_size       ,    is_decoder=UpperCAmelCase       ,    initializer_range=self.initializer_range       ,    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertModel(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase    )
							_UpperCAmelCase									=			model(UpperCAmelCase       ,    token_type_ids=UpperCAmelCase    )
							_UpperCAmelCase									=			model(UpperCAmelCase    )

							self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.seq_length, self.hidden_size)    )
							self.parent.assertEqual(result.pooler_output.shape       ,    (self.batch_size, self.hidden_size)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertForMaskedLM(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    labels=UpperCAmelCase    )
							self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.vocab_size)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertForNextSentencePrediction(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(
							    UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    labels=UpperCAmelCase       ,    )
							self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, 2)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertForPreTraining(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(
							    UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    labels=UpperCAmelCase       ,    next_sentence_label=UpperCAmelCase       ,    )
							self.parent.assertEqual(result.prediction_logits.shape       ,    (self.batch_size, self.seq_length, self.vocab_size)    )
							self.parent.assertEqual(result.seq_relationship_logits.shape       ,    (self.batch_size, 2)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertForQuestionAnswering(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(
							    UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    start_positions=UpperCAmelCase       ,    end_positions=UpperCAmelCase       ,    )
							self.parent.assertEqual(result.start_logits.shape       ,    (self.batch_size, self.seq_length)    )
							self.parent.assertEqual(result.end_logits.shape       ,    (self.batch_size, self.seq_length)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.num_labels
							_UpperCAmelCase									=			MobileBertForSequenceClassification(UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    labels=UpperCAmelCase    )
							self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.num_labels)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.num_labels
							_UpperCAmelCase									=			MobileBertForTokenClassification(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			model(UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    labels=UpperCAmelCase    )
							self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.num_labels)    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.num_choices
							_UpperCAmelCase									=			MobileBertForMultipleChoice(config=UpperCAmelCase    )
							model.to(UpperCAmelCase    )
							model.eval()
							_UpperCAmelCase									=			input_ids.unsqueeze(1    ).expand(-1       ,    self.num_choices       ,    -1    ).contiguous()
							_UpperCAmelCase									=			token_type_ids.unsqueeze(1    ).expand(-1       ,    self.num_choices       ,    -1    ).contiguous()
							_UpperCAmelCase									=			input_mask.unsqueeze(1    ).expand(-1       ,    self.num_choices       ,    -1    ).contiguous()
							_UpperCAmelCase									=			model(
							    UpperCAmelCase       ,    attention_mask=UpperCAmelCase       ,    token_type_ids=UpperCAmelCase       ,    labels=UpperCAmelCase       ,    )
							self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.num_choices)    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.prepare_config_and_inputs()
							(
							    (
							    _UpperCAmelCase
							)					, (
							    _UpperCAmelCase
							)					, (
							    _UpperCAmelCase
							)					, (
							    _UpperCAmelCase
							)					, (
							    _UpperCAmelCase
							)					, (
							    _UpperCAmelCase
							)					, (
							    _UpperCAmelCase
							)					, 
							)									=			config_and_inputs
							_UpperCAmelCase									=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict





@require_torch
class __lowerCamelCase (      snake_case__     ,       snake_case__     ,       unittest.TestCase):





	"""simple docstring"""






	UpperCamelCase__								=      (
	    (
	        MobileBertModel,
	        MobileBertForMaskedLM,
	        MobileBertForMultipleChoice,
	        MobileBertForNextSentencePrediction,
	        MobileBertForPreTraining,
	        MobileBertForQuestionAnswering,
	        MobileBertForSequenceClassification,
	        MobileBertForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	UpperCamelCase__								=      (
	    {
	        "feature-extraction": MobileBertModel,
	        "fill-mask": MobileBertForMaskedLM,
	        "question-answering": MobileBertForQuestionAnswering,
	        "text-classification": MobileBertForSequenceClassification,
	        "token-classification": MobileBertForTokenClassification,
	        "zero-shot": MobileBertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	UpperCamelCase__								=      True

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase=False    ):
							"""simple docstring"""







							_UpperCAmelCase									=			super()._prepare_for_class(UpperCAmelCase       ,    UpperCAmelCase       ,    return_labels=UpperCAmelCase    )

							if return_labels:
													if model_class in get_values(UpperCAmelCase    ):
																			_UpperCAmelCase									=			torch.zeros(
																			    (self.model_tester.batch_size, self.model_tester.seq_length)       ,    dtype=torch.long       ,    device=UpperCAmelCase    )
																			_UpperCAmelCase									=			torch.zeros(
																			    self.model_tester.batch_size       ,    dtype=torch.long       ,    device=UpperCAmelCase    )
							return inputs_dict

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertModelTester(self    )
							_UpperCAmelCase									=			ConfigTester(self       ,    config_class=UpperCAmelCase       ,    hidden_size=37    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							self.config_tester.run_common_tests()

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_model(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_masked_lm(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_multiple_choice(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_pretraining(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_question_answering(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_sequence_classification(*UpperCAmelCase    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_token_classification(*UpperCAmelCase    )



def __A    (      __lowerCAmelCase	)->  Optional[Any]:





						"""simple docstring"""



						return torch.tensor(
						    __lowerCAmelCase ,		dtype=torch.long ,		device=__lowerCAmelCase ,		)


_a      	=  1e-3





@require_torch
@require_sentencepiece
@require_tokenizers
class __lowerCamelCase (      unittest.TestCase):





	"""simple docstring"""






	@slow
	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			MobileBertModel.from_pretrained('google/mobilebert-uncased'    ).to(UpperCAmelCase    )
							_UpperCAmelCase									=			_long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]]    )
							with torch.no_grad():
													_UpperCAmelCase									=			model(UpperCAmelCase    )[0]
							_UpperCAmelCase									=			torch.Size((1, 9, 512)    )
							self.assertEqual(output.shape       ,    UpperCAmelCase    )
							_UpperCAmelCase									=			torch.tensor(
							    [
							        [
							            [-2.4736526e07, 8.2691656e04, 1.6521838e05],
							            [-5.7541704e-01, 3.9056022e00, 4.4011507e00],
							            [2.6047359e00, 1.5677652e00, -1.7324188e-01],
							        ]
							    ]       ,    device=UpperCAmelCase       ,    )

							# MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
							# ~1 difference, it's therefore not a good idea to measure using addition.
							# Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
							# result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
							_UpperCAmelCase									=			torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE    )
							_UpperCAmelCase									=			torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE    )

							self.assertTrue(lower_bound and upper_bound    )

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE										=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE										=						{
    'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/config.json',
    # See all BioGPT models at https://huggingface.co/models?filter=biogpt
}



class 				a							(			__lowerCAmelCase       ):







 """simple docstring"""






 lowerCamelCase	:Any    			=		'''biogpt'''



 def __init__(					self		,				lowerCAmelCase_=4_23_84		,				lowerCAmelCase_=10_24		,				lowerCAmelCase_=24		,				lowerCAmelCase_=16		,				lowerCAmelCase_=40_96		,				lowerCAmelCase_="gelu"		,				lowerCAmelCase_=0.1		,				lowerCAmelCase_=0.1		,				lowerCAmelCase_=10_24		,				lowerCAmelCase_=0.02		,				lowerCAmelCase_=1E-12		,				lowerCAmelCase_=True		,				lowerCAmelCase_=True		,				lowerCAmelCase_=0.0		,				lowerCAmelCase_=0.0		,				lowerCAmelCase_=1		,				lowerCAmelCase_=0		,				lowerCAmelCase_=2		,				**lowerCAmelCase_		,				)  ->					Tuple:
        _A        =  vocab_size
        _A        =  max_position_embeddings
        _A        =  hidden_size
        _A        =  num_hidden_layers
        _A        =  num_attention_heads
        _A        =  intermediate_size
        _A        =  hidden_act
        _A        =  hidden_dropout_prob
        _A        =  attention_probs_dropout_prob
        _A        =  initializer_range
        _A        =  layer_norm_eps
        _A        =  scale_embedding
        _A        =  use_cache
        _A        =  layerdrop
        _A        =  activation_dropout
        super().__init__(pad_token_id=lowerCAmelCase_		,				bos_token_id=lowerCAmelCase_		,				eos_token_id=lowerCAmelCase_		,				**lowerCAmelCase_							)




