




import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           FunnelBaseModel,
           FunnelForMaskedLM,
           FunnelForMultipleChoice,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForSequenceClassification,
           FunnelForTokenClassification,
           FunnelModel,
       )




class 	UpperCamelCase__			:

    '''simple docstring'''



    def __init__(					self  ,						UpperCamelCase__  ,						UpperCamelCase__=13  ,						UpperCamelCase__=7  ,						UpperCamelCase__=True  ,						UpperCamelCase__=True  ,						UpperCamelCase__=True  ,						UpperCamelCase__=True  ,						UpperCamelCase__=99  ,						UpperCamelCase__=[1, 1, 2]  ,						UpperCamelCase__=1  ,						UpperCamelCase__=32  ,						UpperCamelCase__=4  ,						UpperCamelCase__=8  ,						UpperCamelCase__=37  ,						UpperCamelCase__="gelu_new"  ,						UpperCamelCase__=0.1  ,						UpperCamelCase__=0.1  ,						UpperCamelCase__=0.0  ,						UpperCamelCase__=512  ,						UpperCamelCase__=3  ,						UpperCamelCase__=0.02  ,						UpperCamelCase__=3  ,						UpperCamelCase__=4  ,						UpperCamelCase__=None  ,						UpperCamelCase__=False  ,						)  ->   Optional[int]:
           lowerCamelCase     :   Dict    				=      parent
           lowerCamelCase     :   Dict    				=      batch_size
           lowerCamelCase     :   str    				=      seq_length
           lowerCamelCase     :   Optional[Any]    				=      is_training
           lowerCamelCase     :   Any    				=      use_input_mask
           lowerCamelCase     :   int    				=      use_token_type_ids
           lowerCamelCase     :   Tuple    				=      use_labels
           lowerCamelCase     :   Union[str, Any]    				=      vocab_size
           lowerCamelCase     :   Dict    				=      block_sizes
           lowerCamelCase     :   Dict    				=      num_decoder_layers
           lowerCamelCase     :   Optional[int]    				=      d_model
           lowerCamelCase     :   int    				=      n_head
           lowerCamelCase     :   Any    				=      d_head
           lowerCamelCase     :   List[Any]    				=      d_inner
           lowerCamelCase     :   Any    				=      hidden_act
           lowerCamelCase     :   Optional[int]    				=      hidden_dropout
           lowerCamelCase     :   List[str]    				=      attention_dropout
           lowerCamelCase     :   Optional[int]    				=      activation_dropout
           lowerCamelCase     :   Optional[int]    				=      max_position_embeddings
           lowerCamelCase     :   List[Any]    				=      type_vocab_size
           lowerCamelCase     :   List[Any]    				=      2
           lowerCamelCase     :   str    				=      num_labels
           lowerCamelCase     :   List[Any]    				=      num_choices
           lowerCamelCase     :   str    				=      scope
           lowerCamelCase     :   Union[str, Any]    				=      initializer_std

           # Used in the tests to check the size of the first attention layer
           lowerCamelCase     :   Dict    				=      n_head
           # Used in the tests to check the size of the first hidden state
           lowerCamelCase     :   Optional[int]    				=      self.d_model
           # Used in the tests to check the number of output hidden states/attentions
           lowerCamelCase     :   List[Any]    				=      sum(self.block_sizes  ) + (0 if base else self.num_decoder_layers)
           # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
           # the last hidden state of the first block (which is the first hidden state of the decoder).
           if not base:
                  lowerCamelCase     :   Optional[int]    				=      self.num_hidden_layers + 2







    def 					_lowercase  (					self  )  ->   List[str]:
           lowerCamelCase     :   Tuple    				=      ids_tensor([self.batch_size, self.seq_length]  ,						self.vocab_size  )

           lowerCamelCase     :   List[Any]    				=      None
           if self.use_input_mask:
                  lowerCamelCase     :   List[Any]    				=      ids_tensor([self.batch_size, self.seq_length]  ,						vocab_size=2  )

           lowerCamelCase     :   List[str]    				=      None
           if self.use_token_type_ids:
                  lowerCamelCase     :   Union[str, Any]    				=      ids_tensor([self.batch_size, self.seq_length]  ,						self.type_vocab_size  )

           lowerCamelCase     :   Optional[int]    				=      None
           lowerCamelCase     :   Optional[int]    				=      None
           lowerCamelCase     :   Tuple    				=      None
           if self.use_labels:
                  lowerCamelCase     :   Tuple    				=      ids_tensor([self.batch_size]  ,						self.type_sequence_label_size  )
                  lowerCamelCase     :   Union[str, Any]    				=      ids_tensor([self.batch_size, self.seq_length]  ,						self.num_labels  )
                  lowerCamelCase     :   Any    				=      ids_tensor([self.batch_size]  ,						self.num_choices  )
                  lowerCamelCase     :   List[Any]    				=      ids_tensor([self.batch_size, self.seq_length]  ,						1  )

           lowerCamelCase     :   List[Any]    				=      self.get_config()

           return (
               config,
               input_ids,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
               choice_labels,
               fake_token_labels,
           )







    def 					_lowercase  (					self  )  ->   str:
           return FunnelConfig(
               vocab_size=self.vocab_size  ,						block_sizes=self.block_sizes  ,						num_decoder_layers=self.num_decoder_layers  ,						d_model=self.d_model  ,						n_head=self.n_head  ,						d_head=self.d_head  ,						d_inner=self.d_inner  ,						hidden_act=self.hidden_act  ,						hidden_dropout=self.hidden_dropout  ,						attention_dropout=self.attention_dropout  ,						activation_dropout=self.activation_dropout  ,						max_position_embeddings=self.max_position_embeddings  ,						type_vocab_size=self.type_vocab_size  ,						initializer_std=self.initializer_std  ,						)







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   List[str]:
           lowerCamelCase     :   List[Any]    				=      FunnelModel(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   Any    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  )
           lowerCamelCase     :   List[str]    				=      model(UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  )
           lowerCamelCase     :   Optional[Any]    				=      model(UpperCamelCase__  )
           self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)  )

           lowerCamelCase     :   Any    				=      False
           lowerCamelCase     :   Union[str, Any]    				=      model(UpperCamelCase__  )
           self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)  )

           lowerCamelCase     :   Dict    				=      False
           lowerCamelCase     :   int    				=      model(UpperCamelCase__  )
           self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   List[str]:
           lowerCamelCase     :   Optional[int]    				=      FunnelBaseModel(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   Optional[Any]    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  )
           lowerCamelCase     :   List[Any]    				=      model(UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  )
           lowerCamelCase     :   List[str]    				=      model(UpperCamelCase__  )
           self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 2, self.d_model)  )

           lowerCamelCase     :   str    				=      False
           lowerCamelCase     :   Any    				=      model(UpperCamelCase__  )
           self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 3, self.d_model)  )

           lowerCamelCase     :   int    				=      False
           lowerCamelCase     :   Any    				=      model(UpperCamelCase__  )
           self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 2, self.d_model)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   List[Any]:
           lowerCamelCase     :   Optional[int]    				=      self.num_labels
           lowerCamelCase     :   List[str]    				=      FunnelForPreTraining(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   Tuple    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  ,						labels=UpperCamelCase__  )
           self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   int:
           lowerCamelCase     :   Any    				=      FunnelForMaskedLM(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   Dict    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  ,						labels=UpperCamelCase__  )
           self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.vocab_size)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   Tuple:
           lowerCamelCase     :   Union[str, Any]    				=      self.num_labels
           lowerCamelCase     :   Union[str, Any]    				=      FunnelForSequenceClassification(UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   str    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  ,						labels=UpperCamelCase__  )
           self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.num_labels)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   Optional[Any]:
           lowerCamelCase     :   List[str]    				=      self.num_choices
           lowerCamelCase     :   Union[str, Any]    				=      FunnelForMultipleChoice(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   Optional[Any]    				=      input_ids.unsqueeze(1  ).expand(-1  ,						self.num_choices  ,						-1  ).contiguous()
           lowerCamelCase     :   Any    				=      token_type_ids.unsqueeze(1  ).expand(-1  ,						self.num_choices  ,						-1  ).contiguous()
           lowerCamelCase     :   Any    				=      input_mask.unsqueeze(1  ).expand(-1  ,						self.num_choices  ,						-1  ).contiguous()
           lowerCamelCase     :   List[Any]    				=      model(
               UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  ,						labels=UpperCamelCase__  ,						)
           self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.num_choices)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   List[Any]:
           lowerCamelCase     :   Optional[int]    				=      self.num_labels
           lowerCamelCase     :   Any    				=      FunnelForTokenClassification(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   List[Any]    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  ,						labels=UpperCamelCase__  )
           self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.num_labels)  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						)  ->   Union[str, Any]:
           lowerCamelCase     :   Dict    				=      FunnelForQuestionAnswering(config=UpperCamelCase__  )
           model.to(UpperCamelCase__  )
           model.eval()
           lowerCamelCase     :   Tuple    				=      model(
               UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  ,						start_positions=UpperCamelCase__  ,						end_positions=UpperCamelCase__  ,						)
           self.parent.assertEqual(result.start_logits.shape  ,						(self.batch_size, self.seq_length)  )
           self.parent.assertEqual(result.end_logits.shape  ,						(self.batch_size, self.seq_length)  )







    def 					_lowercase  (					self  )  ->   Optional[Any]:
           lowerCamelCase     :   Tuple    				=      self.prepare_config_and_inputs()
           (
               lowerCamelCase   
           )     :   Optional[Any]    				=      config_and_inputs
           lowerCamelCase     :   Tuple    				=      {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
           return config, inputs_dict




@require_torch
class 	UpperCamelCase__			(lowerCAmelCase__					,       lowerCAmelCase__					,       unittest.TestCase  ):

    '''simple docstring'''



    lowerCamelCase_   :			Any			  =   False
    lowerCamelCase_   :			Optional[int]			  =   False
    lowerCamelCase_   :			Optional[Any]			  =   (
        (
            FunnelModel,
            FunnelForMaskedLM,
            FunnelForPreTraining,
            FunnelForQuestionAnswering,
            FunnelForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase_   :			Any			  =   (
        {
            """feature-extraction""": (FunnelBaseModel, FunnelModel),
            """fill-mask""": FunnelForMaskedLM,
            """question-answering""": FunnelForQuestionAnswering,
            """text-classification""": FunnelForSequenceClassification,
            """token-classification""": FunnelForTokenClassification,
            """zero-shot""": FunnelForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__=False  )  ->   Union[str, Any]:
           lowerCamelCase     :   Any    				=      super()._prepare_for_class(UpperCamelCase__  ,						UpperCamelCase__  ,						return_labels=UpperCamelCase__  )

           if return_labels:
                  if model_class in get_values(UpperCamelCase__  ):
                         lowerCamelCase     :   Tuple    				=      torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.seq_length)  ,						dtype=torch.long  ,						device=UpperCamelCase__  )
           return inputs_dict







    def 					_lowercase  (					self  )  ->   str:
           lowerCamelCase     :   Dict    				=      FunnelModelTester(self  )
           lowerCamelCase     :   List[Any]    				=      ConfigTester(self  ,						config_class=UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   Tuple:
           self.config_tester.run_common_tests()







    def 					_lowercase  (					self  )  ->   List[str]:
           lowerCamelCase     :   str    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   int:
           lowerCamelCase     :   Optional[int]    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_pretraining(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   int:
           lowerCamelCase     :   str    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   List[Any]:
           lowerCamelCase     :   List[Any]    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   List[Any]:
           lowerCamelCase     :   Union[str, Any]    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__  )







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   Optional[Any]:
           if hasattr(UpperCamelCase__  ,						"weight"  ) and module.weight is not None:
                  module.weight.data.fill_(3  )
           if hasattr(UpperCamelCase__  ,						"bias"  ) and module.bias is not None:
                  module.bias.data.fill_(3  )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(UpperCamelCase__  ,						UpperCamelCase__  ) and getattr(UpperCamelCase__  ,						UpperCamelCase__  ) is not None:
                         lowerCamelCase     :   Optional[Any]    				=      getattr(UpperCamelCase__  ,						UpperCamelCase__  )
                         weight.data.fill_(3  )




@require_torch
class 	UpperCamelCase__			(lowerCAmelCase__					,       unittest.TestCase  ):

    '''simple docstring'''



    lowerCamelCase_   :			Optional[Any]			  =   False
    lowerCamelCase_   :			Optional[Any]			  =   False
    lowerCamelCase_   :			int			  =   (
        (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
    )







    def 					_lowercase  (					self  )  ->   Tuple:
           lowerCamelCase     :   Dict    				=      FunnelModelTester(self  ,						base=UpperCamelCase__  )
           lowerCamelCase     :   Union[str, Any]    				=      ConfigTester(self  ,						config_class=UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   List[str]:
           self.config_tester.run_common_tests()







    def 					_lowercase  (					self  )  ->   Dict:
           lowerCamelCase     :   Optional[Any]    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_base_model(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   Any:
           lowerCamelCase     :   Optional[int]    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   Union[str, Any]:
           lowerCamelCase     :   Optional[int]    				=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*UpperCamelCase__  )







    def 					_lowercase  (					self  )  ->   str:
           lowerCamelCase     :   Union[str, Any]    				=      self.model_tester.prepare_config_and_inputs_for_common()
           lowerCamelCase     :   Dict    				=      True

           for model_class in self.all_model_classes:
                  if model_class.__name__ == "FunnelBaseModel":
                         continue
                  lowerCamelCase     :   Tuple    				=      model_class(UpperCamelCase__  )
                  model.to(UpperCamelCase__  )
                  model.train()
                  lowerCamelCase     :   Union[str, Any]    				=      self._prepare_for_class(UpperCamelCase__  ,						UpperCamelCase__  ,						return_labels=UpperCamelCase__  )
                  lowerCamelCase     :   Dict    				=      model(**UpperCamelCase__  ).loss
                  loss.backward()







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   Optional[Any]:
           if hasattr(UpperCamelCase__  ,						"weight"  ) and module.weight is not None:
                  module.weight.data.fill_(3  )
           if hasattr(UpperCamelCase__  ,						"bias"  ) and module.bias is not None:
                  module.bias.data.fill_(3  )

           for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(UpperCamelCase__  ,						UpperCamelCase__  ) and getattr(UpperCamelCase__  ,						UpperCamelCase__  ) is not None:
                         lowerCamelCase     :   Dict    				=      getattr(UpperCamelCase__  ,						UpperCamelCase__  )
                         weight.data.fill_(3  )




@require_torch
@require_sentencepiece
@require_tokenizers
class 	UpperCamelCase__			(unittest.TestCase  ):

    '''simple docstring'''



    def 					_lowercase  (					self  )  ->   Union[str, Any]:
           lowerCamelCase     :   int    				=      13
           lowerCamelCase     :   Optional[Any]    				=      7
           lowerCamelCase     :   Any    				=      torch.arange(0  ,						batch_size * sequence_length  ).long().reshape(UpperCamelCase__  ,						UpperCamelCase__  )
           lowerCamelCase     :   List[Any]    				=      [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
           lowerCamelCase     :   Union[str, Any]    				=      torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]  )

           lowerCamelCase     :   Dict    				=      FunnelModel.from_pretrained("sgugger/funnel-random-tiny"  )
           lowerCamelCase     :   Optional[Any]    				=      model(UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  )[0].abs()

           lowerCamelCase     :   int    				=      torch.tensor(2344.8352  )
           lowerCamelCase     :   Optional[Any]    				=      torch.tensor(0.8052  )
           self.assertTrue(torch.allclose(output.sum()  ,						UpperCamelCase__  ,						atol=1e-4  )  )
           self.assertTrue(torch.allclose(output.mean()  ,						UpperCamelCase__  ,						atol=1e-4  )  )

           lowerCamelCase     :   Dict    				=      torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]  )
           lowerCamelCase     :   List[str]    				=      model(UpperCamelCase__  ,						attention_mask=UpperCamelCase__  ,						token_type_ids=UpperCamelCase__  )[0].abs()

           lowerCamelCase     :   Union[str, Any]    				=      torch.tensor(2343.8425  )
           lowerCamelCase     :   Optional[Any]    				=      torch.tensor(0.8049  )
           self.assertTrue(torch.allclose(output.sum()  ,						UpperCamelCase__  ,						atol=1e-4  )  )
           self.assertTrue(torch.allclose(output.mean()  ,						UpperCamelCase__  ,						atol=1e-4  )  )







    @slow
    def 					_lowercase  (					self  )  ->   Any:
           lowerCamelCase     :   List[Any]    				=      FunnelTokenizer.from_pretrained("huggingface/funnel-small"  )
           lowerCamelCase     :   Tuple    				=      FunnelModel.from_pretrained("huggingface/funnel-small"  )
           lowerCamelCase     :   List[Any]    				=      tokenizer("Hello! I am the Funnel Transformer model."  ,						return_tensors="pt"  )
           lowerCamelCase     :   Union[str, Any]    				=      model(**UpperCamelCase__  )[0]

           lowerCamelCase     :   Dict    				=      torch.tensor(235.7246  )
           lowerCamelCase     :   Union[str, Any]    				=      torch.tensor(0.0256  )
           self.assertTrue(torch.allclose(output.sum()  ,						UpperCamelCase__  ,						atol=1e-4  )  )
           self.assertTrue(torch.allclose(output.mean()  ,						UpperCamelCase__  ,						atol=1e-4  )  )












from itertools import compress, repeat
from math import ceil, sqrt

def   A   (   _SCREAMING_SNAKE_CASE					)			->				list[int]:

       if num <= 2:
              return []
       if num == 3:
              return [2]

       # Odd sieve for numbers in range [3, num - 1]
       lowerCamelCase     :   str    				=      bytearray(B"\x01"					) * ((num >> 1) - 1)

       for i in range(3				,int(sqrt(_SCREAMING_SNAKE_CASE					)					) + 1				,2					):
              if sieve[(i >> 1) - 1]:
                     lowerCamelCase     :   Tuple    				=      i**2
                     lowerCamelCase     :   str    				=      repeat(
                         0				,ceil((num - i_squared) / (i << 1)					)					)

       return [2] + list(compress(range(3				,_SCREAMING_SNAKE_CASE				,2					)				,_SCREAMING_SNAKE_CASE					)					)


if __name__ == "__main__":
       import doctest

       doctest.testmod()






