



"""simple docstring"""






import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       NystromformerForMaskedLM,
       NystromformerForMultipleChoice,
       NystromformerForQuestionAnswering,
       NystromformerForSequenceClassification,
       NystromformerForTokenClassification,
       NystromformerModel,
   )
   from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST






class        __snake_case       :




  def __init__( self     :						List[Any]						,					__lowerCAmelCase     :						Optional[int]						,					__lowerCAmelCase     :						List[Any]=1_3						,					__lowerCAmelCase     :						Any=7						,					__lowerCAmelCase     :						List[Any]=True						,					__lowerCAmelCase     :						Dict=True						,					__lowerCAmelCase     :						List[Any]=True						,					__lowerCAmelCase     :						Optional[int]=True						,					__lowerCAmelCase     :						int=9_9						,					__lowerCAmelCase     :						str=3_2						,					__lowerCAmelCase     :						Tuple=5						,					__lowerCAmelCase     :						Optional[Any]=4						,					__lowerCAmelCase     :						Optional[Any]=3_7						,					__lowerCAmelCase     :						Optional[Any]="gelu"						,					__lowerCAmelCase     :						int=0.1						,					__lowerCAmelCase     :						Any=0.1						,					__lowerCAmelCase     :						int=5_1_2						,					__lowerCAmelCase     :						Union[str, Any]=1_6						,					__lowerCAmelCase     :						int=2						,					__lowerCAmelCase     :						List[str]=0.02						,					__lowerCAmelCase     :						Optional[int]=3						,					__lowerCAmelCase     :						Optional[int]=4						,					__lowerCAmelCase     :						Dict=None						,					):






         """simple docstring"""






         _lowerCamelCase						:				Tuple    			= parent
         _lowerCamelCase						:				int    			= batch_size
         _lowerCamelCase						:				List[Any]    			= seq_length
         _lowerCamelCase						:				Tuple    			= is_training
         _lowerCamelCase						:				int    			= use_input_mask
         _lowerCamelCase						:				Dict    			= use_token_type_ids
         _lowerCamelCase						:				Optional[int]    			= use_labels
         _lowerCamelCase						:				Dict    			= vocab_size
         _lowerCamelCase						:				List[str]    			= hidden_size
         _lowerCamelCase						:				Optional[int]    			= num_hidden_layers
         _lowerCamelCase						:				Tuple    			= num_attention_heads
         _lowerCamelCase						:				Optional[int]    			= intermediate_size
         _lowerCamelCase						:				Tuple    			= hidden_act
         _lowerCamelCase						:				List[Any]    			= hidden_dropout_prob
         _lowerCamelCase						:				Union[str, Any]    			= attention_probs_dropout_prob
         _lowerCamelCase						:				Tuple    			= max_position_embeddings
         _lowerCamelCase						:				Dict    			= type_vocab_size
         _lowerCamelCase						:				Dict    			= type_sequence_label_size
         _lowerCamelCase						:				str    			= initializer_range
         _lowerCamelCase						:				int    			= num_labels
         _lowerCamelCase						:				str    			= num_choices
         _lowerCamelCase						:				str    			= scope




  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:				List[str]    			= ids_tensor([self.batch_size, self.seq_length]						,					self.vocab_size		)

         _lowerCamelCase						:				Union[str, Any]    			= None
         if self.use_input_mask:
                _lowerCamelCase						:				Dict    			= random_attention_mask([self.batch_size, self.seq_length]		)

         _lowerCamelCase						:				Optional[int]    			= None
         if self.use_token_type_ids:
                _lowerCamelCase						:				Any    			= ids_tensor([self.batch_size, self.seq_length]						,					self.type_vocab_size		)

         _lowerCamelCase						:				Optional[int]    			= None
         _lowerCamelCase						:				int    			= None
         _lowerCamelCase						:				Optional[int]    			= None
         if self.use_labels:
                _lowerCamelCase						:				Optional[int]    			= ids_tensor([self.batch_size]						,					self.type_sequence_label_size		)
                _lowerCamelCase						:				Dict    			= ids_tensor([self.batch_size, self.seq_length]						,					self.num_labels		)
                _lowerCamelCase						:				Dict    			= ids_tensor([self.batch_size]						,					self.num_choices		)

         _lowerCamelCase						:				Union[str, Any]    			= self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         return NystromformerConfig(
             vocab_size=self.vocab_size						,					hidden_size=self.hidden_size						,					num_hidden_layers=self.num_hidden_layers						,					num_attention_heads=self.num_attention_heads						,					intermediate_size=self.intermediate_size						,					hidden_act=self.hidden_act						,					hidden_dropout_prob=self.hidden_dropout_prob						,					attention_probs_dropout_prob=self.attention_probs_dropout_prob						,					max_position_embeddings=self.max_position_embeddings						,					type_vocab_size=self.type_vocab_size						,					is_decoder=__lowerCAmelCase						,					initializer_range=self.initializer_range						,					)




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]						,					__lowerCAmelCase     :						List[str]						,					__lowerCAmelCase     :						Optional[int]						,					__lowerCAmelCase     :						Optional[int]						,					__lowerCAmelCase     :						List[Any]						,					__lowerCAmelCase     :						int						,					__lowerCAmelCase     :						Optional[Any]						,					__lowerCAmelCase     :						str		):






         """simple docstring"""






         _lowerCamelCase						:				Dict    			= NystromformerModel(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:				int    			= model(__lowerCAmelCase						,					attention_mask=__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase		)
         _lowerCamelCase						:				Any    			= model(__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase		)
         _lowerCamelCase						:				str    			= model(__lowerCAmelCase		)
         self.parent.assertEqual(result.last_hidden_state.shape						,					(self.batch_size, self.seq_length, self.hidden_size)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Any						,					__lowerCAmelCase     :						int						,					__lowerCAmelCase     :						Optional[int]						,					__lowerCAmelCase     :						Dict						,					__lowerCAmelCase     :						Any						,					__lowerCAmelCase     :						Dict						,					__lowerCAmelCase     :						Union[str, Any]						,					__lowerCAmelCase     :						Dict		):






         """simple docstring"""






         _lowerCamelCase						:				int    			= NystromformerForMaskedLM(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:				Optional[int]    			= model(__lowerCAmelCase						,					attention_mask=__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase						,					labels=__lowerCAmelCase		)
         self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.seq_length, self.vocab_size)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						str						,					__lowerCAmelCase     :						Union[str, Any]						,					__lowerCAmelCase     :						str						,					__lowerCAmelCase     :						Optional[int]						,					__lowerCAmelCase     :						List[Any]						,					__lowerCAmelCase     :						Tuple						,					__lowerCAmelCase     :						Dict						,					__lowerCAmelCase     :						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				Dict    			= NystromformerForQuestionAnswering(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:				List[Any]    			= model(
             __lowerCAmelCase						,					attention_mask=__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase						,					start_positions=__lowerCAmelCase						,					end_positions=__lowerCAmelCase						,					)
         self.parent.assertEqual(result.start_logits.shape						,					(self.batch_size, self.seq_length)		)
         self.parent.assertEqual(result.end_logits.shape						,					(self.batch_size, self.seq_length)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						int						,					__lowerCAmelCase     :						Tuple						,					__lowerCAmelCase     :						List[str]						,					__lowerCAmelCase     :						str						,					__lowerCAmelCase     :						List[Any]						,					__lowerCAmelCase     :						Optional[int]						,					__lowerCAmelCase     :						int						,					__lowerCAmelCase     :						str		):






         """simple docstring"""






         _lowerCamelCase						:				int    			= self.num_labels
         _lowerCamelCase						:				Union[str, Any]    			= NystromformerForSequenceClassification(__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:				Tuple    			= model(__lowerCAmelCase						,					attention_mask=__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase						,					labels=__lowerCAmelCase		)
         self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.num_labels)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						str						,					__lowerCAmelCase     :						Union[str, Any]						,					__lowerCAmelCase     :						Any						,					__lowerCAmelCase     :						Tuple						,					__lowerCAmelCase     :						Any						,					__lowerCAmelCase     :						Union[str, Any]						,					__lowerCAmelCase     :						List[str]						,					__lowerCAmelCase     :						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				str    			= self.num_labels
         _lowerCamelCase						:				int    			= NystromformerForTokenClassification(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:				Tuple    			= model(__lowerCAmelCase						,					attention_mask=__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase						,					labels=__lowerCAmelCase		)
         self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.seq_length, self.num_labels)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Union[str, Any]						,					__lowerCAmelCase     :						Union[str, Any]						,					__lowerCAmelCase     :						int						,					__lowerCAmelCase     :						Tuple						,					__lowerCAmelCase     :						Dict						,					__lowerCAmelCase     :						Any						,					__lowerCAmelCase     :						Any						,					__lowerCAmelCase     :						Tuple		):






         """simple docstring"""






         _lowerCamelCase						:				List[Any]    			= self.num_choices
         _lowerCamelCase						:				str    			= NystromformerForMultipleChoice(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:				str    			= input_ids.unsqueeze(1		).expand(-1						,					self.num_choices						,					-1		).contiguous()
         _lowerCamelCase						:				int    			= token_type_ids.unsqueeze(1		).expand(-1						,					self.num_choices						,					-1		).contiguous()
         _lowerCamelCase						:				Tuple    			= input_mask.unsqueeze(1		).expand(-1						,					self.num_choices						,					-1		).contiguous()
         _lowerCamelCase						:				int    			= model(
             __lowerCAmelCase						,					attention_mask=__lowerCAmelCase						,					token_type_ids=__lowerCAmelCase						,					labels=__lowerCAmelCase						,					)
         self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.num_choices)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[int]		):






         """simple docstring"""






         _lowerCamelCase						:				List[str]    			= self.prepare_config_and_inputs()
         (
             (
             _lowerCamelCase
         )			,    (
             _lowerCamelCase
         )			,    (
             _lowerCamelCase
         )			,    (
             _lowerCamelCase
         )			,    (
             _lowerCamelCase
         )			,    (
             _lowerCamelCase
         )			,    (
             _lowerCamelCase
         )			,    
         )						:				List[Any]    			= config_and_inputs
         _lowerCamelCase						:				Optional[int]    			= {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict






@require_torch
class        __snake_case       (	_lowercase  ,     _lowercase  ,     unittest.TestCase):
  snake_case__				:   List[str]  			=						(
      (
          NystromformerModel,
          NystromformerForMaskedLM,
          NystromformerForMultipleChoice,
          NystromformerForQuestionAnswering,
          NystromformerForSequenceClassification,
          NystromformerForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  snake_case__				:   Optional[Any]  			=						(
      {
          "feature-extraction": NystromformerModel,
          "fill-mask": NystromformerForMaskedLM,
          "question-answering": NystromformerForQuestionAnswering,
          "text-classification": NystromformerForSequenceClassification,
          "token-classification": NystromformerForTokenClassification,
          "zero-shot": NystromformerForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  snake_case__				:   Optional[int]  			=						False
  snake_case__				:   int  			=						False




  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:				Any    			= NystromformerModelTester(self		)
         _lowerCamelCase						:				List[str]    			= ConfigTester(self						,					config_class=__lowerCAmelCase						,					hidden_size=3_7		)




  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         self.config_tester.run_common_tests()




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[int]		):






         """simple docstring"""






         _lowerCamelCase						:				Optional[Any]    			= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:				Dict    			= self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
                _lowerCamelCase						:				str    			= type
                self.model_tester.create_and_check_model(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				List[Any]    			= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Any		):






         """simple docstring"""






         _lowerCamelCase						:				Any    			= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						int		):






         """simple docstring"""






         _lowerCamelCase						:				Any    			= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Tuple		):






         """simple docstring"""






         _lowerCamelCase						:				int    			= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Union[str, Any]		):






         """simple docstring"""






         _lowerCamelCase						:				List[str]    			= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*__lowerCAmelCase		)




  @slow
  def 						SCREAMING_SNAKE_CASE				( self     :						List[Any]		):






         """simple docstring"""






         for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                _lowerCamelCase						:				Tuple    			= NystromformerModel.from_pretrained(__lowerCAmelCase		)
                self.assertIsNotNone(__lowerCAmelCase		)






@require_torch
class        __snake_case       (	unittest.TestCase):




  @slow
  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[int]		):






         """simple docstring"""






         _lowerCamelCase						:				Tuple    			= NystromformerModel.from_pretrained('''uw-madison/nystromformer-512'''		)
         _lowerCamelCase						:				List[Any]    			= torch.tensor([[0, 1, 2, 3, 4, 5]]		)

         with torch.no_grad():
                _lowerCamelCase						:				Dict    			= model(__lowerCAmelCase		)[0]

         _lowerCamelCase						:				str    			= torch.Size((1, 6, 7_6_8)		)
         self.assertEqual(output.shape						,					__lowerCAmelCase		)

         _lowerCamelCase						:				int    			= torch.tensor(
             [[[-0.45_32, -0.09_36, 0.51_37], [-0.26_76, 0.06_28, 0.61_86], [-0.36_29, -0.17_26, 0.47_16]]]		)

         self.assertTrue(torch.allclose(output[:, :3, :3]						,					__lowerCAmelCase						,					atol=1E-4		)		)




  @slow
  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:				Dict    			= '''the [MASK] of Belgium is Brussels'''

         _lowerCamelCase						:				str    			= AutoTokenizer.from_pretrained('''uw-madison/nystromformer-512'''		)
         _lowerCamelCase						:				Any    			= NystromformerForMaskedLM.from_pretrained('''uw-madison/nystromformer-512'''		)

         _lowerCamelCase						:				Tuple    			= tokenizer(__lowerCAmelCase						,					return_tensors='''pt'''		)

         with torch.no_grad():
                _lowerCamelCase						:				List[Any]    			= model(encoding.input_ids		).logits

         _lowerCamelCase						:				str    			= token_logits[:, 2, :].argmax(-1		)[0]

         self.assertEqual(tokenizer.decode(__lowerCAmelCase		)						,					'''capital'''		)









"""simple docstring"""



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a_ =			logging.get_logger(__name__)

a_ =			{
    """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/config.json""",
    """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/config.json""",
    """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/config.json""",
    """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/config.json""",
    """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/config.json""",
    """roberta-large-openai-detector""": """https://huggingface.co/roberta-large-openai-detector/resolve/main/config.json""",
}






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 _lowerCamelCase        =     """roberta"""

 def __init__(    self					,						__lowerCamelCase=5_0265					,						__lowerCamelCase=768					,						__lowerCamelCase=12					,						__lowerCamelCase=12					,						__lowerCamelCase=3072					,						__lowerCamelCase="gelu"					,						__lowerCamelCase=0.1					,						__lowerCamelCase=0.1					,						__lowerCamelCase=512					,						__lowerCamelCase=2					,						__lowerCamelCase=0.0_2					,						__lowerCamelCase=1e-1_2					,						__lowerCamelCase=1					,						__lowerCamelCase=0					,						__lowerCamelCase=2					,						__lowerCamelCase="absolute"					,						__lowerCamelCase=True					,						__lowerCamelCase=None					,						**__lowerCamelCase					,						):



     '''simple docstring'''




     super().__init__(pad_token_id=__lowerCamelCase					,						bos_token_id=__lowerCamelCase					,						eos_token_id=__lowerCamelCase					,						**__lowerCamelCase  )

     __A    :						str						 =     vocab_size
     __A    :						Any						 =     hidden_size
     __A    :						Any						 =     num_hidden_layers
     __A    :						int						 =     num_attention_heads
     __A    :						List[Any]						 =     hidden_act
     __A    :						Any						 =     intermediate_size
     __A    :						List[str]						 =     hidden_dropout_prob
     __A    :						List[Any]						 =     attention_probs_dropout_prob
     __A    :						Optional[Any]						 =     max_position_embeddings
     __A    :						str						 =     type_vocab_size
     __A    :						int						 =     initializer_range
     __A    :						Tuple						 =     layer_norm_eps
     __A    :						str						 =     position_embedding_type
     __A    :						int						 =     use_cache
     __A    :						Optional[int]						 =     classifier_dropout






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 @property
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     if self.task == "multiple-choice":
         __A    :						List[Any]						 =     {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
     else:
         __A    :						Optional[Any]						 =     {0: '''batch''', 1: '''sequence'''}
     return OrderedDict(
         [
             ('''input_ids''', dynamic_axis),
             ('''attention_mask''', dynamic_axis),
         ]  )




