code: 
  
  
  
  '''simple docstring'''
  
  
  
  import argparse
  
  import torch
  
  from diffusers import UNetaDConditionModel
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A							,__A							,__A=None							,__A=None							,__A=None       ):
  
  
  
   '''simple docstring'''
  
  
  
   assert isinstance(__A							,__A       ), "Paths should be a list of dicts containing 'old' and 'new' keys."
  
   # Splits the attention layers into three variables.
   if attention_paths_to_split is not None:
    for path, path_map in attention_paths_to_split.items():
     __UpperCamelCase				      =  old_checkpoint[path]
     __UpperCamelCase				      =  old_tensor.shape[0] // 3
  
     __UpperCamelCase				      =  (-1, channels) if len(old_tensor.shape       ) == 3 else (-1)
  
     __UpperCamelCase				      =  old_tensor.shape[0] // config["""num_head_channels"""] // 3
  
     __UpperCamelCase				      =  old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]       )
     __UpperCamelCase      ,		__UpperCamelCase      ,		__UpperCamelCase				      =  old_tensor.split(channels // num_heads							,dim=1       )
  
     __UpperCamelCase				      =  query.reshape(__A       )
     __UpperCamelCase				      =  key.reshape(__A       )
     __UpperCamelCase				      =  value.reshape(__A       )
  
   for path in paths:
    __UpperCamelCase				      =  path["""new"""]
  
    # These have already been assigned
    if attention_paths_to_split is not None and new_path in attention_paths_to_split:
     continue
  
    if additional_replacements is not None:
     for replacement in additional_replacements:
      __UpperCamelCase				      =  new_path.replace(replacement["""old"""]							,replacement["""new"""]       )
  
          # proj_attn.weight has to be converted from conv 1D to linear
    __UpperCamelCase				      =  old_checkpoint[path["""old"""]]
    __UpperCamelCase				      =  ["""proj_attn.weight"""]
    __UpperCamelCase				      =  ["""proj_out.weight""", """proj_in.weight"""]
    if any(k in new_path for k in names       ):
     __UpperCamelCase				      =  weight[:, :, 0]
    elif any(k in new_path for k in names_a       ) and len(weight.shape       ) > 2 and ".attentions." not in new_path:
     __UpperCamelCase				      =  weight[:, :, 0]
    else:
     __UpperCamelCase				      =  weight
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A=0       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  []
   for old_item in old_list:
    __UpperCamelCase				      =  old_item
  
    #         new_item = new_item.replace('norm.weight', 'group_norm.weight')
    #         new_item = new_item.replace('norm.bias', 'group_norm.bias')
  
    #         new_item = new_item.replace('proj_out.weight', 'proj_attn.weight')
    #         new_item = new_item.replace('proj_out.bias', 'proj_attn.bias')
  
    #         new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
  
    mapping.append({"""old""": old_item, """new""": new_item}       )
  
   return mapping
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A=1       ):
  
  
  
   '''simple docstring'''
  
  
  
   if n_shave_prefix_segments >= 0:
    return ".".join(path.split("""."""       )[n_shave_prefix_segments:]       )
   else:
    return ".".join(path.split("""."""       )[:n_shave_prefix_segments]       )
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A=0       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  []
   for old_item in old_list:
    mapping.append({"""old""": old_item, """new""": old_item}       )
  
   return mapping
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A=0       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  []
   for old_item in old_list:
    __UpperCamelCase				      =  old_item.replace("""in_layers.0"""							,"""norm1"""       )
    __UpperCamelCase				      =  new_item.replace("""in_layers.2"""							,"""conv1"""       )
  
    __UpperCamelCase				      =  new_item.replace("""out_layers.0"""							,"""norm2"""       )
    __UpperCamelCase				      =  new_item.replace("""out_layers.3"""							,"""conv2"""       )
  
    __UpperCamelCase				      =  new_item.replace("""emb_layers.1"""							,"""time_emb_proj"""       )
    __UpperCamelCase				      =  new_item.replace("""skip_connection"""							,"""conv_shortcut"""       )
  
    __UpperCamelCase				      =  shave_segments(__A							,n_shave_prefix_segments=__A       )
  
    if "temopral_conv" not in old_item:
     mapping.append({"""old""": old_item, """new""": new_item}       )
  
   return mapping
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A							,__A=None							,__A=False       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  {}
   __UpperCamelCase				      =  list(checkpoint.keys()       )
  
   __UpperCamelCase				      =  """model.diffusion_model."""
  
   # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA
   if sum(k.startswith("""model_ema"""       ) for k in keys       ) > 100 and extract_ema:
    print(f"Checkpoint {path} has both EMA and non-EMA weights."       )
    print(
        """In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA"""
        """ weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag."""       )
    for key in keys:
     if key.startswith("""model.diffusion_model"""       ):
      __UpperCamelCase				      =  """model_ema.""" + """""".join(key.split("""."""       )[1:]       )
      __UpperCamelCase				      =  checkpoint.pop(__A       )
   else:
    if sum(k.startswith("""model_ema"""       ) for k in keys       ) > 100:
     print(
         """In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA"""
         """ weights (usually better for inference), please make sure to add the `--extract_ema` flag."""       )
  
    for key in keys:
     __UpperCamelCase				      =  checkpoint.pop(__A       )
  
   __UpperCamelCase				      =  {}
  
   __UpperCamelCase				      =  unet_state_dict["""time_embed.0.weight"""]
   __UpperCamelCase				      =  unet_state_dict["""time_embed.0.bias"""]
   __UpperCamelCase				      =  unet_state_dict["""time_embed.2.weight"""]
   __UpperCamelCase				      =  unet_state_dict["""time_embed.2.bias"""]
  
   if config["class_embed_type"] is None:
    # No parameters to port
    ...
   elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection":
    __UpperCamelCase				      =  unet_state_dict["""label_emb.0.0.weight"""]
    __UpperCamelCase				      =  unet_state_dict["""label_emb.0.0.bias"""]
    __UpperCamelCase				      =  unet_state_dict["""label_emb.0.2.weight"""]
    __UpperCamelCase				      =  unet_state_dict["""label_emb.0.2.bias"""]
   else:
    raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}"       )
  
   __UpperCamelCase				      =  unet_state_dict["""input_blocks.0.0.weight"""]
   __UpperCamelCase				      =  unet_state_dict["""input_blocks.0.0.bias"""]
  
   __UpperCamelCase				      =  [v for v in unet_state_dict if v.startswith("""input_blocks.0.1"""       )]
   __UpperCamelCase				      =  renew_attention_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """input_blocks.0.1""", """new""": """transformer_in"""}
   assign_to_checkpoint(__A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
   __UpperCamelCase				      =  unet_state_dict["""out.0.weight"""]
   __UpperCamelCase				      =  unet_state_dict["""out.0.bias"""]
   __UpperCamelCase				      =  unet_state_dict["""out.2.weight"""]
   __UpperCamelCase				      =  unet_state_dict["""out.2.bias"""]
  
   # Retrieves the keys for the input blocks only
   __UpperCamelCase				      =  len({""".""".join(layer.split("""."""       )[:2]       ) for layer in unet_state_dict if """input_blocks""" in layer}       )
   __UpperCamelCase				      =  {
       layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key]
       for layer_id in range(__A       )
   }
  
   # Retrieves the keys for the middle blocks only
   __UpperCamelCase				      =  len({""".""".join(layer.split("""."""       )[:2]       ) for layer in unet_state_dict if """middle_block""" in layer}       )
   __UpperCamelCase				      =  {
       layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key]
       for layer_id in range(__A       )
   }
  
   # Retrieves the keys for the output blocks only
   __UpperCamelCase				      =  len({""".""".join(layer.split("""."""       )[:2]       ) for layer in unet_state_dict if """output_blocks""" in layer}       )
   __UpperCamelCase				      =  {
       layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key]
       for layer_id in range(__A       )
   }
  
   for i in range(1							,__A       ):
    __UpperCamelCase				      =  (i - 1) // (config["""layers_per_block"""] + 1)
    __UpperCamelCase				      =  (i - 1) % (config["""layers_per_block"""] + 1)
  
    __UpperCamelCase				      =  [
        key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key
    ]
    __UpperCamelCase				      =  [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key]
    __UpperCamelCase				      =  [key for key in input_blocks[i] if f"input_blocks.{i}.2" in key]
  
    if f"input_blocks.{i}.op.weight" in unet_state_dict:
     __UpperCamelCase				      =  unet_state_dict.pop(
         f"input_blocks.{i}.op.weight"       )
     __UpperCamelCase				      =  unet_state_dict.pop(
         f"input_blocks.{i}.op.bias"       )
  
    __UpperCamelCase				      =  renew_resnet_paths(__A       )
    __UpperCamelCase				      =  {"""old""": f"input_blocks.{i}.0", """new""": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"}
    assign_to_checkpoint(
        __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
    __UpperCamelCase				      =  [key for key in resnets if """temopral_conv""" in key]
    __UpperCamelCase				      =  renew_temp_conv_paths(__A       )
    __UpperCamelCase				      =  {
        """old""": f"input_blocks.{i}.0.temopral_conv",
        """new""": f"down_blocks.{block_id}.temp_convs.{layer_in_block_id}",
    }
    assign_to_checkpoint(
        __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
    if len(__A       ):
     __UpperCamelCase				      =  renew_attention_paths(__A       )
     __UpperCamelCase				      =  {"""old""": f"input_blocks.{i}.1", """new""": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"}
     assign_to_checkpoint(
         __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
    if len(__A       ):
     __UpperCamelCase				      =  renew_attention_paths(__A       )
     __UpperCamelCase				      =  {
         """old""": f"input_blocks.{i}.2",
         """new""": f"down_blocks.{block_id}.temp_attentions.{layer_in_block_id}",
     }
     assign_to_checkpoint(
         __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
   __UpperCamelCase				      =  middle_blocks[0]
   __UpperCamelCase				      =  [key for key in resnet_a if """temopral_conv""" in key]
   __UpperCamelCase				      =  middle_blocks[1]
   __UpperCamelCase				      =  middle_blocks[2]
   __UpperCamelCase				      =  middle_blocks[3]
   __UpperCamelCase				      =  [key for key in resnet_a if """temopral_conv""" in key]
  
   __UpperCamelCase				      =  renew_resnet_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """middle_block.0""", """new""": """mid_block.resnets.0"""}
   assign_to_checkpoint(
       __A							,__A							,__A							,config=__A							,additional_replacements=[meta_path]       )
  
   __UpperCamelCase				      =  renew_temp_conv_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """middle_block.0.temopral_conv""", """new""": """mid_block.temp_convs.0"""}
   assign_to_checkpoint(
       __A							,__A							,__A							,config=__A							,additional_replacements=[meta_path]       )
  
   __UpperCamelCase				      =  renew_resnet_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """middle_block.3""", """new""": """mid_block.resnets.1"""}
   assign_to_checkpoint(
       __A							,__A							,__A							,config=__A							,additional_replacements=[meta_path]       )
  
   __UpperCamelCase				      =  renew_temp_conv_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """middle_block.3.temopral_conv""", """new""": """mid_block.temp_convs.1"""}
   assign_to_checkpoint(
       __A							,__A							,__A							,config=__A							,additional_replacements=[meta_path]       )
  
   __UpperCamelCase				      =  renew_attention_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """middle_block.1""", """new""": """mid_block.attentions.0"""}
   assign_to_checkpoint(
       __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
   __UpperCamelCase				      =  renew_attention_paths(__A       )
   __UpperCamelCase				      =  {"""old""": """middle_block.2""", """new""": """mid_block.temp_attentions.0"""}
   assign_to_checkpoint(
       __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
   for i in range(__A       ):
    __UpperCamelCase				      =  i // (config["""layers_per_block"""] + 1)
    __UpperCamelCase				      =  i % (config["""layers_per_block"""] + 1)
    __UpperCamelCase				      =  [shave_segments(__A							,2       ) for name in output_blocks[i]]
    __UpperCamelCase				      =  {}
  
    for layer in output_block_layers:
     __UpperCamelCase      ,		__UpperCamelCase				      =  layer.split("""."""       )[0], shave_segments(__A							,1       )
     if layer_id in output_block_list:
      output_block_list[layer_id].append(__A       )
     else:
      __UpperCamelCase				      =  [layer_name]
  
    if len(__A       ) > 1:
     __UpperCamelCase				      =  [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key]
     __UpperCamelCase				      =  [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key]
     __UpperCamelCase				      =  [key for key in output_blocks[i] if f"output_blocks.{i}.2" in key]
  
     __UpperCamelCase				      =  renew_resnet_paths(__A       )
     __UpperCamelCase				      =  renew_resnet_paths(__A       )
  
     __UpperCamelCase				      =  {"""old""": f"output_blocks.{i}.0", """new""": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"}
     assign_to_checkpoint(
         __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
     __UpperCamelCase				      =  [key for key in resnets if """temopral_conv""" in key]
     __UpperCamelCase				      =  renew_temp_conv_paths(__A       )
     __UpperCamelCase				      =  {
         """old""": f"output_blocks.{i}.0.temopral_conv",
         """new""": f"up_blocks.{block_id}.temp_convs.{layer_in_block_id}",
     }
     assign_to_checkpoint(
         __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
     __UpperCamelCase				      =  {k: sorted(__A       ) for k, v in output_block_list.items()}
     if ["conv.bias", "conv.weight"] in output_block_list.values():
      __UpperCamelCase				      =  list(output_block_list.values()       ).index(["""conv.bias""", """conv.weight"""]       )
      __UpperCamelCase				      =  unet_state_dict[
          f"output_blocks.{i}.{index}.conv.weight"
      ]
      __UpperCamelCase				      =  unet_state_dict[
          f"output_blocks.{i}.{index}.conv.bias"
      ]
  
      # Clear attentions as they have been attributed above.
      if len(__A       ) == 2:
       __UpperCamelCase				      =  []
  
     if len(__A       ):
      __UpperCamelCase				      =  renew_attention_paths(__A       )
      __UpperCamelCase				      =  {
          """old""": f"output_blocks.{i}.1",
          """new""": f"up_blocks.{block_id}.attentions.{layer_in_block_id}",
      }
      assign_to_checkpoint(
          __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
  
     if len(__A       ):
      __UpperCamelCase				      =  renew_attention_paths(__A       )
      __UpperCamelCase				      =  {
          """old""": f"output_blocks.{i}.2",
          """new""": f"up_blocks.{block_id}.temp_attentions.{layer_in_block_id}",
      }
      assign_to_checkpoint(
          __A							,__A							,__A							,additional_replacements=[meta_path]							,config=__A       )
    else:
     __UpperCamelCase				      =  renew_resnet_paths(__A							,n_shave_prefix_segments=1       )
     for path in resnet_0_paths:
      __UpperCamelCase				      =  """.""".join(["""output_blocks""", str(__A       ), path["""old"""]]       )
      __UpperCamelCase				      =  """.""".join(["""up_blocks""", str(__A       ), """resnets""", str(__A       ), path["""new"""]]       )
      __UpperCamelCase				      =  unet_state_dict[old_path]
  
     __UpperCamelCase				      =  [l for l in output_block_layers if """temopral_conv""" in l]
     for path in temopral_conv_paths:
      __UpperCamelCase				      =  path.split("""temopral_conv."""       )[-1]
      __UpperCamelCase				      =  """.""".join(["""output_blocks""", str(__A       ), str(__A       ), """temopral_conv""", pruned_path]       )
      __UpperCamelCase				      =  """.""".join(["""up_blocks""", str(__A       ), """temp_convs""", str(__A       ), pruned_path]       )
      __UpperCamelCase				      =  unet_state_dict[old_path]
  
   return new_checkpoint
  
  
  if __name__ == "__main__":
   a__						:			str										=			argparse.ArgumentParser()
  
   parser.add_argument(
       '--checkpoint_path', default=None, type=str, required=True, help='Path to the checkpoint to convert.'
   )
   parser.add_argument('--dump_path', default=None, type=str, required=True, help='Path to the output model.')
   a__						:			List[Any]										=			parser.parse_args()
  
   a__						:			int										=			torch.load(args.checkpoint_path, map_location='cpu')
   a__						:			int										=			UNetaDConditionModel()
  
   a__						:			List[str]										=			convert_ldm_unet_checkpoint(unet_checkpoint, unet.config)
  
   a__						:			List[Any]										=			set(unet.state_dict().keys()) - set(converted_ckpt.keys())
   a__						:			List[str]										=			set(converted_ckpt.keys()) - set(unet.state_dict().keys())
  
   assert len(diff_a) == len(diff_a) == 0, "Converted weights don't match"
  
   # load state_dict
   unet.load_state_dict(converted_ckpt)
  
   unet.save_pretrained(args.dump_path)
  
   # -- finish converting the unet --
  
  
  
  
  
  
style_context: 
  
  
  
  '''simple docstring'''
  
  
  
  import unittest
  
  from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
   import torch
  
   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       FunnelBaseModel,
       FunnelForMaskedLM,
       FunnelForMultipleChoice,
       FunnelForPreTraining,
       FunnelForQuestionAnswering,
       FunnelForSequenceClassification,
       FunnelForTokenClassification,
       FunnelModel,
   )
  class        UpperCAmelCase__  :
  
  
  
   def __init__(  self		,	lowercase		,	lowercase=1_3		,	lowercase=7		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=9_9		,	lowercase=[1, 1, 2]		,	lowercase=1		,	lowercase=3_2		,	lowercase=4		,	lowercase=8		,	lowercase=3_7		,	lowercase="gelu_new"		,	lowercase=0.1		,	lowercase=0.1		,	lowercase=0.0		,	lowercase=5_1_2		,	lowercase=3		,	lowercase=0.02		,	lowercase=3		,	lowercase=4		,	lowercase=None		,	lowercase=False		,	)  ->							Any:
    __UpperCamelCase				      =  parent
    __UpperCamelCase				      =  batch_size
    __UpperCamelCase				      =  seq_length
    __UpperCamelCase				      =  is_training
    __UpperCamelCase				      =  use_input_mask
    __UpperCamelCase				      =  use_token_type_ids
    __UpperCamelCase				      =  use_labels
    __UpperCamelCase				      =  vocab_size
    __UpperCamelCase				      =  block_sizes
    __UpperCamelCase				      =  num_decoder_layers
    __UpperCamelCase				      =  d_model
    __UpperCamelCase				      =  n_head
    __UpperCamelCase				      =  d_head
    __UpperCamelCase				      =  d_inner
    __UpperCamelCase				      =  hidden_act
    __UpperCamelCase				      =  hidden_dropout
    __UpperCamelCase				      =  attention_dropout
    __UpperCamelCase				      =  activation_dropout
    __UpperCamelCase				      =  max_position_embeddings
    __UpperCamelCase				      =  type_vocab_size
    __UpperCamelCase				      =  2
    __UpperCamelCase				      =  num_labels
    __UpperCamelCase				      =  num_choices
    __UpperCamelCase				      =  scope
    __UpperCamelCase				      =  initializer_std
  
    # Used in the tests to check the size of the first attention layer
    __UpperCamelCase				      =  n_head
    # Used in the tests to check the size of the first hidden state
    __UpperCamelCase				      =  self.d_model
    # Used in the tests to check the number of output hidden states/attentions
    __UpperCamelCase				      =  sum(self.block_sizes			) + (0 if base else self.num_decoder_layers)
    # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
    # the last hidden state of the first block (which is the first hidden state of the decoder).
    if not base:
     __UpperCamelCase				      =  self.num_hidden_layers + 2
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size			)
  
    __UpperCamelCase				      =  None
    if self.use_input_mask:
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	vocab_size=2			)
  
    __UpperCamelCase				      =  None
    if self.use_token_type_ids:
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size			)
  
    __UpperCamelCase				      =  None
    __UpperCamelCase				      =  None
    __UpperCamelCase				      =  None
    if self.use_labels:
     __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.type_sequence_label_size			)
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.num_labels			)
     __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.num_choices			)
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	1			)
  
    __UpperCamelCase				      =  self.get_config()
  
    return (
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
        fake_token_labels,
    )
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    return FunnelConfig(
        vocab_size=self.vocab_size		,	block_sizes=self.block_sizes		,	num_decoder_layers=self.num_decoder_layers		,	d_model=self.d_model		,	n_head=self.n_head		,	d_head=self.d_head		,	d_inner=self.d_inner		,	hidden_act=self.hidden_act		,	hidden_dropout=self.hidden_dropout		,	attention_dropout=self.attention_dropout		,	activation_dropout=self.activation_dropout		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	initializer_std=self.initializer_std		,	)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
    __UpperCamelCase				      =  FunnelModel(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							str:
    __UpperCamelCase				      =  FunnelBaseModel(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 3, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
    __UpperCamelCase				      =  self.num_labels
    __UpperCamelCase				      =  FunnelForPreTraining(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
    __UpperCamelCase				      =  FunnelForMaskedLM(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.vocab_size)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
    __UpperCamelCase				      =  self.num_labels
    __UpperCamelCase				      =  FunnelForSequenceClassification(lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_labels)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							int:
    __UpperCamelCase				      =  self.num_choices
    __UpperCamelCase				      =  FunnelForMultipleChoice(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  input_ids.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
    __UpperCamelCase				      =  token_type_ids.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
    __UpperCamelCase				      =  input_mask.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
    __UpperCamelCase				      =  model(
        lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase		,	)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_choices)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
    __UpperCamelCase				      =  self.num_labels
    __UpperCamelCase				      =  FunnelForTokenClassification(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.num_labels)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
    __UpperCamelCase				      =  FunnelForQuestionAnswering(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(
        lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	start_positions=lowercase		,	end_positions=lowercase		,	)
    self.parent.assertEqual(result.start_logits.shape		,	(self.batch_size, self.seq_length)			)
    self.parent.assertEqual(result.end_logits.shape		,	(self.batch_size, self.seq_length)			)
  
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    __UpperCamelCase				      =  self.prepare_config_and_inputs()
    (
        (
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		
    )				      =  config_and_inputs
    __UpperCamelCase				      =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
    return config, inputs_dict
  
  
  @require_torch
  class        UpperCAmelCase__  (    UpperCAmelCase_							,			UpperCAmelCase_							,			unittest.TestCase):
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				(
       (
           FunnelModel,
           FunnelForMaskedLM,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   __SCREAMING_SNAKE_CASE					      =				(
       {
           '''feature-extraction''': (FunnelBaseModel, FunnelModel),
           '''fill-mask''': FunnelForMaskedLM,
           '''question-answering''': FunnelForQuestionAnswering,
           '''text-classification''': FunnelForSequenceClassification,
           '''token-classification''': FunnelForTokenClassification,
           '''zero-shot''': FunnelForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase=False			)  ->							Tuple:
    __UpperCamelCase				      =  super()._prepare_for_class(lowercase		,	lowercase		,	return_labels=lowercase			)
  
    if return_labels:
     if model_class in get_values(lowercase			):
      __UpperCamelCase				      =  torch.zeros(
          (self.model_tester.batch_size, self.model_tester.seq_length)		,	dtype=torch.long		,	device=lowercase			)
    return inputs_dict
  
  
  
   def    __lowerCamelCase				(  self			)  ->							List[str]:
    __UpperCamelCase				      =  FunnelModelTester(self			)
    __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    self.config_tester.run_common_tests()
  
  
  
   def    __lowerCamelCase				(  self			)  ->							List[str]:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_pretraining(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							str:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Dict:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*lowercase			)
  
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							int:
    if hasattr(lowercase		,	"""weight"""			) and module.weight is not None:
     module.weight.data.fill_(3			)
    if hasattr(lowercase		,	"""bias"""			) and module.bias is not None:
     module.bias.data.fill_(3			)
  
    for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
     if hasattr(lowercase		,	lowercase			) and getattr(lowercase		,	lowercase			) is not None:
      __UpperCamelCase				      =  getattr(lowercase		,	lowercase			)
      weight.data.fill_(3			)
  
  
  @require_torch
  class        UpperCAmelCase__  (    UpperCAmelCase_							,			unittest.TestCase):
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				(
       (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
   )
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  FunnelModelTester(self		,	base=lowercase			)
    __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    self.config_tester.run_common_tests()
  
  
  
   def    __lowerCamelCase				(  self			)  ->							str:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_base_model(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							str:
    __UpperCamelCase      ,		__UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs_for_common()
    __UpperCamelCase				      =  True
  
    for model_class in self.all_model_classes:
     if model_class.__name__ == "FunnelBaseModel":
      continue
     __UpperCamelCase				      =  model_class(lowercase			)
     model.to(lowercase			)
     model.train()
     __UpperCamelCase				      =  self._prepare_for_class(lowercase		,	lowercase		,	return_labels=lowercase			)
     __UpperCamelCase				      =  model(**lowercase			).loss
     loss.backward()
  
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[Any]:
    if hasattr(lowercase		,	"""weight"""			) and module.weight is not None:
     module.weight.data.fill_(3			)
    if hasattr(lowercase		,	"""bias"""			) and module.bias is not None:
     module.bias.data.fill_(3			)
  
    for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
     if hasattr(lowercase		,	lowercase			) and getattr(lowercase		,	lowercase			) is not None:
      __UpperCamelCase				      =  getattr(lowercase		,	lowercase			)
      weight.data.fill_(3			)
  
  
  @require_torch
  @require_sentencepiece
  @require_tokenizers
  class        UpperCAmelCase__  (    unittest.TestCase):
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    __UpperCamelCase				      =  1_3
    __UpperCamelCase				      =  7
    __UpperCamelCase				      =  torch.arange(0		,	batch_size * sequence_length			).long().reshape(lowercase		,	lowercase			)
    __UpperCamelCase				      =  [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
    __UpperCamelCase				      =  torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]			)
  
    __UpperCamelCase				      =  FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""			)
    __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)[0].abs()
  
    __UpperCamelCase				      =  torch.tensor(2_344.8_352			)
    __UpperCamelCase				      =  torch.tensor(0.8_052			)
    self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
    self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)
  
    __UpperCamelCase				      =  torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]			)
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)[0].abs()
  
    __UpperCamelCase				      =  torch.tensor(2_343.8_425			)
    __UpperCamelCase				      =  torch.tensor(0.8_049			)
    self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
    self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)
  
  
  
  
   @slow
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""			)
    __UpperCamelCase				      =  FunnelModel.from_pretrained("""huggingface/funnel-small"""			)
    __UpperCamelCase				      =  tokenizer("""Hello! I am the Funnel Transformer model."""		,	return_tensors="""pt"""			)
    __UpperCamelCase				      =  model(**lowercase			)[0]
  
    __UpperCamelCase				      =  torch.tensor(235.7_246			)
    __UpperCamelCase				      =  torch.tensor(0.0_256			)
    self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
    self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)
  
  
  
  
  
  