






"""simple docstring"""





import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      YosoForMaskedLM,
      YosoForMultipleChoice,
      YosoForQuestionAnswering,
      YosoForSequenceClassification,
      YosoForTokenClassification,
      YosoModel,
  )
  from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST
class  __lowerCamelCase     :

   def __init__(       self			,					snake_case_			,					snake_case_=13			,					snake_case_=7			,					snake_case_=True			,					snake_case_=True			,					snake_case_=True			,					snake_case_=True			,					snake_case_=99			,					snake_case_=32			,					snake_case_=5			,					snake_case_=4			,					snake_case_=37			,					snake_case_="gelu"			,					snake_case_=0.1			,					snake_case_=0.1			,					snake_case_=512			,					snake_case_=16			,					snake_case_=2			,					snake_case_=0.02			,					snake_case_=3			,					snake_case_=4			,					snake_case_=None			,					)		->							List[Any]:
       UpperCamelCase__								=     parent
       UpperCamelCase__								=     batch_size
       UpperCamelCase__								=     seq_length
       UpperCamelCase__								=     is_training
       UpperCamelCase__								=     use_input_mask
       UpperCamelCase__								=     use_token_type_ids
       UpperCamelCase__								=     use_labels
       UpperCamelCase__								=     vocab_size
       UpperCamelCase__								=     hidden_size
       UpperCamelCase__								=     num_hidden_layers
       UpperCamelCase__								=     num_attention_heads
       UpperCamelCase__								=     intermediate_size
       UpperCamelCase__								=     hidden_act
       UpperCamelCase__								=     hidden_dropout_prob
       UpperCamelCase__								=     attention_probs_dropout_prob
       UpperCamelCase__								=     max_position_embeddings
       UpperCamelCase__								=     type_vocab_size
       UpperCamelCase__								=     type_sequence_label_size
       UpperCamelCase__								=     initializer_range
       UpperCamelCase__								=     num_labels
       UpperCamelCase__								=     num_choices
       UpperCamelCase__								=     scope

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Dict:
       UpperCamelCase__								=     ids_tensor([self.batch_size, self.seq_length]			,					self.vocab_size   )

       UpperCamelCase__								=     None
       if self.use_input_mask:
           UpperCamelCase__								=     random_attention_mask([self.batch_size, self.seq_length]   )

       UpperCamelCase__								=     None
       if self.use_token_type_ids:
           UpperCamelCase__								=     ids_tensor([self.batch_size, self.seq_length]			,					self.type_vocab_size   )

       UpperCamelCase__								=     None
       UpperCamelCase__								=     None
       UpperCamelCase__								=     None
       if self.use_labels:
           UpperCamelCase__								=     ids_tensor([self.batch_size]			,					self.type_sequence_label_size   )
           UpperCamelCase__								=     ids_tensor([self.batch_size, self.seq_length]			,					self.num_labels   )
           UpperCamelCase__								=     ids_tensor([self.batch_size]			,					self.num_choices   )

       UpperCamelCase__								=     self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       return YosoConfig(
           vocab_size=self.vocab_size			,					hidden_size=self.hidden_size			,					num_hidden_layers=self.num_hidden_layers			,					num_attention_heads=self.num_attention_heads			,					intermediate_size=self.intermediate_size			,					hidden_act=self.hidden_act			,					hidden_dropout_prob=self.hidden_dropout_prob			,					attention_probs_dropout_prob=self.attention_probs_dropout_prob			,					max_position_embeddings=self.max_position_embeddings			,					type_vocab_size=self.type_vocab_size			,					is_decoder=snake_case_			,					initializer_range=self.initializer_range			,					)

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Any:
       UpperCamelCase__								=     self.get_config()
       UpperCamelCase__								=     300
       return config

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       (
           (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    
       )								=     self.prepare_config_and_inputs()

       UpperCamelCase__								=     True
       UpperCamelCase__								=     floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
       UpperCamelCase__								=     ids_tensor([self.batch_size, self.seq_length]			,					vocab_size=2   )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_   )		->							List[Any]:
       UpperCamelCase__								=     YosoModel(config=snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     model(snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_   )
       UpperCamelCase__								=     model(snake_case_			,					token_type_ids=snake_case_   )
       UpperCamelCase__								=     model(snake_case_   )
       self.parent.assertEqual(result.last_hidden_state.shape			,					(self.batch_size, self.seq_length, self.hidden_size)   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					)		->							List[Any]:
       UpperCamelCase__								=     True
       UpperCamelCase__								=     YosoModel(snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     model(
           snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					encoder_hidden_states=snake_case_			,					encoder_attention_mask=snake_case_			,					)
       UpperCamelCase__								=     model(
           snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					encoder_hidden_states=snake_case_			,					)
       UpperCamelCase__								=     model(snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_   )
       self.parent.assertEqual(result.last_hidden_state.shape			,					(self.batch_size, self.seq_length, self.hidden_size)   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_   )		->							str:
       UpperCamelCase__								=     YosoForMaskedLM(config=snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     model(snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					labels=snake_case_   )
       self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.vocab_size)   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_   )		->							Union[str, Any]:
       UpperCamelCase__								=     YosoForQuestionAnswering(config=snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     model(
           snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					start_positions=snake_case_			,					end_positions=snake_case_			,					)
       self.parent.assertEqual(result.start_logits.shape			,					(self.batch_size, self.seq_length)   )
       self.parent.assertEqual(result.end_logits.shape			,					(self.batch_size, self.seq_length)   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_   )		->							List[Any]:
       UpperCamelCase__								=     self.num_labels
       UpperCamelCase__								=     YosoForSequenceClassification(snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     model(snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					labels=snake_case_   )
       self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.num_labels)   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_   )		->							Optional[int]:
       UpperCamelCase__								=     self.num_labels
       UpperCamelCase__								=     YosoForTokenClassification(config=snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     model(snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					labels=snake_case_   )
       self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.seq_length, self.num_labels)   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_   )		->							Dict:
       UpperCamelCase__								=     self.num_choices
       UpperCamelCase__								=     YosoForMultipleChoice(config=snake_case_   )
       model.to(snake_case_   )
       model.eval()
       UpperCamelCase__								=     input_ids.unsqueeze(1   ).expand(-1			,					self.num_choices			,					-1   ).contiguous()
       UpperCamelCase__								=     token_type_ids.unsqueeze(1   ).expand(-1			,					self.num_choices			,					-1   ).contiguous()
       UpperCamelCase__								=     input_mask.unsqueeze(1   ).expand(-1			,					self.num_choices			,					-1   ).contiguous()
       UpperCamelCase__								=     model(
           snake_case_			,					attention_mask=snake_case_			,					token_type_ids=snake_case_			,					labels=snake_case_			,					)
       self.parent.assertEqual(result.logits.shape			,					(self.batch_size, self.num_choices)   )







   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Optional[int]:
       UpperCamelCase__								=     self.prepare_config_and_inputs()
       (
           (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    (
           UpperCamelCase__
       )							,    
       )								=     config_and_inputs
       UpperCamelCase__								=     {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
       return config, inputs_dict







@require_torch
class  __lowerCamelCase     (	_a				,						_a				,						unittest.TestCase	):
   a   :					Optional[Any]			    =(
       (
           YosoModel,
           YosoForMaskedLM,
           YosoForMultipleChoice,
           YosoForQuestionAnswering,
           YosoForSequenceClassification,
           YosoForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   a   :					Tuple			    =False
   a   :					List[str]			    =False
   a   :					Dict			    =False

   a   :					Optional[int]			    =()
   a   :					List[Any]			    =(
       {
           """feature-extraction""": YosoModel,
           """fill-mask""": YosoForMaskedLM,
           """question-answering""": YosoForQuestionAnswering,
           """text-classification""": YosoForSequenceClassification,
           """token-classification""": YosoForTokenClassification,
           """zero-shot""": YosoForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							int:
       UpperCamelCase__								=     YosoModelTester(self   )
       UpperCamelCase__								=     ConfigTester(self			,					config_class=snake_case_			,					hidden_size=37   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       self.config_tester.run_common_tests()

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Tuple:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
           UpperCamelCase__								=     type
           self.model_tester.create_and_check_model(*snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Tuple:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       UpperCamelCase__								=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*snake_case_   )

   @slow
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           UpperCamelCase__								=     YosoModel.from_pretrained(snake_case_   )
           self.assertIsNotNone(snake_case_   )







   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Any:
       return







@require_torch
class  __lowerCamelCase     (	unittest.TestCase	):

   @slow
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Optional[Any]:
       UpperCamelCase__								=     YosoModel.from_pretrained('uw-madison/yoso-4096'   )
       UpperCamelCase__								=     torch.tensor([[0, 1, 2, 3, 4, 5]]   )

       with torch.no_grad():
           UpperCamelCase__								=     model(snake_case_   )[0]

       UpperCamelCase__								=     torch.Size((1, 6, 768)   )
       self.assertEqual(output.shape			,					snake_case_   )

       UpperCamelCase__								=     torch.tensor(
           [[[-0.0_611, 0.1_242, 0.0_840], [0.0_280, -0.0_048, 0.1_125], [0.0_106, 0.0_226, 0.0_751]]]   )

       self.assertTrue(torch.allclose(output[:, :3, :3]			,					snake_case_			,					atol=1E-4   )   )

   @slow
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       UpperCamelCase__								=     YosoForMaskedLM.from_pretrained('uw-madison/yoso-4096'   )
       UpperCamelCase__								=     torch.tensor([[0, 1, 2, 3, 4, 5]]   )

       with torch.no_grad():
           UpperCamelCase__								=     model(snake_case_   )[0]

       UpperCamelCase__								=     5_0265

       UpperCamelCase__								=     torch.Size((1, 6, vocab_size)   )
       self.assertEqual(output.shape			,					snake_case_   )

       UpperCamelCase__								=     torch.tensor(
           [[[-2.1_313, -3.7_285, -2.2_407], [-2.7_047, -3.3_314, -2.6_408], [0.0_629, -2.5_166, -0.3_356]]]   )

       self.assertTrue(torch.allclose(output[:, :3, :3]			,					snake_case_			,					atol=1E-4   )   )







   @slow
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							int:
       UpperCamelCase__								=     YosoForMaskedLM.from_pretrained('uw-madison/yoso-4096'   )
       UpperCamelCase__								=     torch.arange(4096   ).unsqueeze(0   )

       with torch.no_grad():
           UpperCamelCase__								=     model(snake_case_   )[0]

       UpperCamelCase__								=     5_0265

       UpperCamelCase__								=     torch.Size((1, 4096, vocab_size)   )
       self.assertEqual(output.shape			,					snake_case_   )

       UpperCamelCase__								=     torch.tensor(
           [[[-2.3_914, -4.3_742, -5.0_956], [-4.0_988, -4.2_384, -7.0_406], [-3.1_427, -3.7_192, -6.6_800]]]   )

       self.assertTrue(torch.allclose(output[:, :3, :3]			,					snake_case_			,					atol=1E-4   )   )












"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__       :			Union[str, Any]=						logging.get_logger(__name__)

A__       :			Dict=						{
    """google/rembert""": """https://huggingface.co/google/rembert/resolve/main/config.json""",
    # See all RemBERT models at https://huggingface.co/models?filter=rembert
}
class  __lowerCamelCase     (	_a	):
   a   :					Optional[int]			    ="""rembert"""

   def __init__(       self			,					snake_case_=25_0300			,					snake_case_=1152			,					snake_case_=32			,					snake_case_=18			,					snake_case_=256			,					snake_case_=1664			,					snake_case_=4608			,					snake_case_="gelu"			,					snake_case_=0.0			,					snake_case_=0.0			,					snake_case_=0.1			,					snake_case_=512			,					snake_case_=2			,					snake_case_=0.02			,					snake_case_=1E-12			,					snake_case_=True			,					snake_case_=0			,					snake_case_=312			,					snake_case_=313			,					**snake_case_			,					)		->							str:
       super().__init__(pad_token_id=snake_case_			,					bos_token_id=snake_case_			,					eos_token_id=snake_case_			,					**snake_case_   )

       UpperCamelCase__								=     vocab_size
       UpperCamelCase__								=     input_embedding_size
       UpperCamelCase__								=     output_embedding_size
       UpperCamelCase__								=     max_position_embeddings
       UpperCamelCase__								=     hidden_size
       UpperCamelCase__								=     num_hidden_layers
       UpperCamelCase__								=     num_attention_heads
       UpperCamelCase__								=     intermediate_size
       UpperCamelCase__								=     hidden_act
       UpperCamelCase__								=     hidden_dropout_prob
       UpperCamelCase__								=     attention_probs_dropout_prob
       UpperCamelCase__								=     classifier_dropout_prob
       UpperCamelCase__								=     initializer_range
       UpperCamelCase__								=     type_vocab_size
       UpperCamelCase__								=     layer_norm_eps
       UpperCamelCase__								=     use_cache
       UpperCamelCase__								=     False







class  __lowerCamelCase     (	_a	):

   @property
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Mapping[str, Mapping[int, str]]:
       if self.task == "multiple-choice":
           UpperCamelCase__								=     {0: 'batch', 1: 'choice', 2: 'sequence'}
       else:
           UpperCamelCase__								=     {0: 'batch', 1: 'sequence'}
       return OrderedDict(
           [
               ('input_ids', dynamic_axis),
               ('attention_mask', dynamic_axis),
               ('token_type_ids', dynamic_axis),
           ]   )







   @property
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							float:
       return 1E-4




