code: 
  
  
  
  '''simple docstring'''
  
  
  
  import json
  import os
  from typing import TYPE_CHECKING, List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation
  
  a__						:			Dict										=			logging.get_logger(__name__)
  
  a__						:			str										=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}
  
  a__						:			List[str]										=			{
      'vocab_file': {
          'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
          'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
          'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
          'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
          'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
          'microsoft/deberta-xlarge-mnli': (
              'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
          ),
      },
      'merges_file': {
          'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
          'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
          'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
          'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
          'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
          'microsoft/deberta-xlarge-mnli': (
              'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
          ),
      },
  }
  
  a__						:			List[str]										=			{
      'microsoft/deberta-base': 5_1_2,
      'microsoft/deberta-large': 5_1_2,
      'microsoft/deberta-xlarge': 5_1_2,
      'microsoft/deberta-base-mnli': 5_1_2,
      'microsoft/deberta-large-mnli': 5_1_2,
      'microsoft/deberta-xlarge-mnli': 5_1_2,
  }
  
  a__						:			Any										=			{
      'microsoft/deberta-base': {'do_lower_case': False},
      'microsoft/deberta-large': {'do_lower_case': False},
  }
  
  
  
  
  
  
  
  def _lowercase   (  ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  (
       list(range(ord("""!"""       )							,ord("""~"""       ) + 1       )       ) + list(range(ord("""¡"""       )							,ord("""¬"""       ) + 1       )       ) + list(range(ord("""®"""       )							,ord("""ÿ"""       ) + 1       )       )
   )
   __UpperCamelCase				      =  bs[:]
   __UpperCamelCase				      =  0
   for b in range(2**8       ):
    if b not in bs:
     bs.append(__A       )
     cs.append(2**8 + n       )
     n += 1
   __UpperCamelCase				      =  [chr(__A       ) for n in cs]
   return dict(zip(__A							,__A       )       )
  
  
  
  
  
  
  
  def _lowercase   (  __A       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  set()
   __UpperCamelCase				      =  word[0]
   for char in word[1:]:
    pairs.add((prev_char, char)       )
    __UpperCamelCase				      =  char
   return pairs
  
  
  
  
  
  
  class        UpperCAmelCase__  (    UpperCAmelCase_):
   __SCREAMING_SNAKE_CASE					      =				VOCAB_FILES_NAMES
   __SCREAMING_SNAKE_CASE					      =				PRETRAINED_VOCAB_FILES_MAP
   __SCREAMING_SNAKE_CASE					      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __SCREAMING_SNAKE_CASE					      =				['''input_ids''', '''attention_mask''', '''token_type_ids''']
  
  
  
   def __init__(  self		,	lowercase		,	lowercase		,	lowercase="replace"		,	lowercase="[CLS]"		,	lowercase="[SEP]"		,	lowercase="[SEP]"		,	lowercase="[CLS]"		,	lowercase="[UNK]"		,	lowercase="[PAD]"		,	lowercase="[MASK]"		,	lowercase=False		,	lowercase=False		,	**lowercase		,	)  ->							int:
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else bos_token
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else eos_token
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else sep_token
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else cls_token
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else unk_token
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else pad_token
  
    # Mask token behave like a normal word, i.e. include the space before it
    __UpperCamelCase				      =  AddedToken(lowercase		,	lstrip=lowercase		,	rstrip=lowercase			) if isinstance(lowercase		,	lowercase			) else mask_token
  
    super().__init__(
        errors=lowercase		,	bos_token=lowercase		,	eos_token=lowercase		,	unk_token=lowercase		,	sep_token=lowercase		,	cls_token=lowercase		,	pad_token=lowercase		,	mask_token=lowercase		,	add_prefix_space=lowercase		,	add_bos_token=lowercase		,	**lowercase		,	)
    __UpperCamelCase				      =  add_bos_token
  
    with open(lowercase		,	encoding="""utf-8"""			) as vocab_handle:
     __UpperCamelCase				      =  json.load(lowercase			)
    __UpperCamelCase				      =  {v: k for k, v in self.encoder.items()}
    __UpperCamelCase				      =  errors  # how to handle errors in decoding
    __UpperCamelCase				      =  bytes_to_unicode()
    __UpperCamelCase				      =  {v: k for k, v in self.byte_encoder.items()}
    with open(lowercase		,	encoding="""utf-8"""			) as merges_handle:
     __UpperCamelCase				      =  merges_handle.read().split("""\n"""			)[1:-1]
    __UpperCamelCase				      =  [tuple(merge.split()			) for merge in bpe_merges]
    __UpperCamelCase				      =  dict(zip(lowercase		,	range(len(lowercase			)			)			)			)
    __UpperCamelCase				      =  {}
    __UpperCamelCase				      =  add_prefix_space
  
    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __UpperCamelCase				      =  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)
  
  
  
   @property
   # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
   def    __lowerCamelCase				(  self			)  ->							str:
    return len(self.encoder			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[Any]:
    return dict(self.encoder		,	**self.added_tokens_encoder			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							Tuple:
    if token in self.cache:
     return self.cache[token]
    __UpperCamelCase				      =  tuple(lowercase			)
    __UpperCamelCase				      =  get_pairs(lowercase			)
  
    if not pairs:
     return token
  
    while True:
     __UpperCamelCase				      =  min(lowercase		,	key=lambda lowercase			: self.bpe_ranks.get(lowercase		,	float("""inf"""			)			)			)
     if bigram not in self.bpe_ranks:
      break
     __UpperCamelCase      ,		__UpperCamelCase				      =  bigram
     __UpperCamelCase				      =  []
     __UpperCamelCase				      =  0
     while i < len(lowercase			):
      try:
       __UpperCamelCase				      =  word.index(lowercase		,	lowercase			)
      except ValueError:
       new_word.extend(word[i:]			)
       break
      else:
       new_word.extend(word[i:j]			)
       __UpperCamelCase				      =  j
  
      if word[i] == first and i < len(lowercase			) - 1 and word[i + 1] == second:
       new_word.append(first + second			)
       i += 2
      else:
       new_word.append(word[i]			)
       i += 1
     __UpperCamelCase				      =  tuple(lowercase			)
     __UpperCamelCase				      =  new_word
     if len(lowercase			) == 1:
      break
     else:
      __UpperCamelCase				      =  get_pairs(lowercase			)
    __UpperCamelCase				      =  """ """.join(lowercase			)
    __UpperCamelCase				      =  word
    return word
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase = None			)  ->							List[int]:
    if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    __UpperCamelCase				      =  [self.cls_token_id]
    __UpperCamelCase				      =  [self.sep_token_id]
    return cls + token_ids_a + sep + token_ids_a + sep
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase = None		,	lowercase = False			)  ->							List[int]:
    if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=lowercase		,	token_ids_a=lowercase		,	already_has_special_tokens=lowercase			)
  
    if token_ids_a is None:
     return [1] + ([0] * len(lowercase			)) + [1]
    return [1] + ([0] * len(lowercase			)) + [1] + ([0] * len(lowercase			)) + [1]
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase = None			)  ->							List[int]:
    __UpperCamelCase				      =  [self.sep_token_id]
    __UpperCamelCase				      =  [self.cls_token_id]
  
    if token_ids_a is None:
     return len(cls + token_ids_a + sep			) * [0]
    return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							Union[str, Any]:
    __UpperCamelCase				      =  []
    for token in re.findall(self.pat		,	lowercase			):
     __UpperCamelCase				      =  """""".join(
         self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase			).split(""" """			)			)
    return bpe_tokens
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[str]:
    return self.encoder.get(lowercase		,	self.encoder.get(self.unk_token			)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							Any:
    return self.decoder.get(lowercase			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[Any]:
    __UpperCamelCase				      =  """""".join(lowercase			)
    __UpperCamelCase				      =  bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8"""		,	errors=self.errors			)
    return text
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase = None			)  ->							Tuple[str]:
    if not os.path.isdir(lowercase			):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
     return
    __UpperCamelCase				      =  os.path.join(
        lowercase		,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
    __UpperCamelCase				      =  os.path.join(
        lowercase		,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)
  
    with open(lowercase		,	"""w"""		,	encoding="""utf-8"""			) as f:
     f.write(json.dumps(self.encoder		,	indent=2		,	sort_keys=lowercase		,	ensure_ascii=lowercase			) + """\n"""			)
  
    __UpperCamelCase				      =  0
    with open(lowercase		,	"""w"""		,	encoding="""utf-8"""			) as writer:
     writer.write("""#version: 0.2\n"""			)
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,	key=lambda lowercase			: kv[1]			):
      if index != token_index:
       logger.warning(
           f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
           """ Please check that the tokenizer is not corrupted!"""			)
       __UpperCamelCase				      =  token_index
      writer.write(""" """.join(lowercase			) + """\n"""			)
      index += 1
  
    return vocab_file, merge_file
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase=False		,	**lowercase			)  ->							Optional[int]:
    __UpperCamelCase				      =  kwargs.pop("""add_prefix_space"""		,	self.add_prefix_space			)
    if (is_split_into_words or add_prefix_space) and (len(lowercase			) > 0 and not text[0].isspace()):
     __UpperCamelCase				      =  """ """ + text
    return (text, kwargs)
  
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[int]:
    __UpperCamelCase				      =  []
    for is_user, text in conversation.iter_texts():
     input_ids.extend(self.encode(lowercase		,	add_special_tokens=lowercase			) + [self.eos_token_id]			)
    if len(lowercase			) > self.model_max_length:
     __UpperCamelCase				      =  input_ids[-self.model_max_length :]
    return input_ids
  
  
  
  
  
  
style_context: 
  
  
  
  '''simple docstring'''
  
  
  
  import unittest
  
  from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
   import torch
  
   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       FunnelBaseModel,
       FunnelForMaskedLM,
       FunnelForMultipleChoice,
       FunnelForPreTraining,
       FunnelForQuestionAnswering,
       FunnelForSequenceClassification,
       FunnelForTokenClassification,
       FunnelModel,
   )
  class        UpperCAmelCase__  :
  
  
  
   def __init__(  self		,	lowercase		,	lowercase=1_3		,	lowercase=7		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=9_9		,	lowercase=[1, 1, 2]		,	lowercase=1		,	lowercase=3_2		,	lowercase=4		,	lowercase=8		,	lowercase=3_7		,	lowercase="gelu_new"		,	lowercase=0.1		,	lowercase=0.1		,	lowercase=0.0		,	lowercase=5_1_2		,	lowercase=3		,	lowercase=0.02		,	lowercase=3		,	lowercase=4		,	lowercase=None		,	lowercase=False		,	)  ->							Any:
    __UpperCamelCase				      =  parent
    __UpperCamelCase				      =  batch_size
    __UpperCamelCase				      =  seq_length
    __UpperCamelCase				      =  is_training
    __UpperCamelCase				      =  use_input_mask
    __UpperCamelCase				      =  use_token_type_ids
    __UpperCamelCase				      =  use_labels
    __UpperCamelCase				      =  vocab_size
    __UpperCamelCase				      =  block_sizes
    __UpperCamelCase				      =  num_decoder_layers
    __UpperCamelCase				      =  d_model
    __UpperCamelCase				      =  n_head
    __UpperCamelCase				      =  d_head
    __UpperCamelCase				      =  d_inner
    __UpperCamelCase				      =  hidden_act
    __UpperCamelCase				      =  hidden_dropout
    __UpperCamelCase				      =  attention_dropout
    __UpperCamelCase				      =  activation_dropout
    __UpperCamelCase				      =  max_position_embeddings
    __UpperCamelCase				      =  type_vocab_size
    __UpperCamelCase				      =  2
    __UpperCamelCase				      =  num_labels
    __UpperCamelCase				      =  num_choices
    __UpperCamelCase				      =  scope
    __UpperCamelCase				      =  initializer_std
  
    # Used in the tests to check the size of the first attention layer
    __UpperCamelCase				      =  n_head
    # Used in the tests to check the size of the first hidden state
    __UpperCamelCase				      =  self.d_model
    # Used in the tests to check the number of output hidden states/attentions
    __UpperCamelCase				      =  sum(self.block_sizes			) + (0 if base else self.num_decoder_layers)
    # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
    # the last hidden state of the first block (which is the first hidden state of the decoder).
    if not base:
     __UpperCamelCase				      =  self.num_hidden_layers + 2
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size			)
  
    __UpperCamelCase				      =  None
    if self.use_input_mask:
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	vocab_size=2			)
  
    __UpperCamelCase				      =  None
    if self.use_token_type_ids:
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size			)
  
    __UpperCamelCase				      =  None
    __UpperCamelCase				      =  None
    __UpperCamelCase				      =  None
    if self.use_labels:
     __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.type_sequence_label_size			)
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.num_labels			)
     __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.num_choices			)
     __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	1			)
  
    __UpperCamelCase				      =  self.get_config()
  
    return (
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
        fake_token_labels,
    )
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    return FunnelConfig(
        vocab_size=self.vocab_size		,	block_sizes=self.block_sizes		,	num_decoder_layers=self.num_decoder_layers		,	d_model=self.d_model		,	n_head=self.n_head		,	d_head=self.d_head		,	d_inner=self.d_inner		,	hidden_act=self.hidden_act		,	hidden_dropout=self.hidden_dropout		,	attention_dropout=self.attention_dropout		,	activation_dropout=self.activation_dropout		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	initializer_std=self.initializer_std		,	)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
    __UpperCamelCase				      =  FunnelModel(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							str:
    __UpperCamelCase				      =  FunnelBaseModel(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 3, self.d_model)			)
  
    __UpperCamelCase				      =  False
    __UpperCamelCase				      =  model(lowercase			)
    self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
    __UpperCamelCase				      =  self.num_labels
    __UpperCamelCase				      =  FunnelForPreTraining(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
    __UpperCamelCase				      =  FunnelForMaskedLM(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.vocab_size)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
    __UpperCamelCase				      =  self.num_labels
    __UpperCamelCase				      =  FunnelForSequenceClassification(lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_labels)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							int:
    __UpperCamelCase				      =  self.num_choices
    __UpperCamelCase				      =  FunnelForMultipleChoice(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  input_ids.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
    __UpperCamelCase				      =  token_type_ids.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
    __UpperCamelCase				      =  input_mask.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
    __UpperCamelCase				      =  model(
        lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase		,	)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_choices)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
    __UpperCamelCase				      =  self.num_labels
    __UpperCamelCase				      =  FunnelForTokenClassification(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
    self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.num_labels)			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
    __UpperCamelCase				      =  FunnelForQuestionAnswering(config=lowercase			)
    model.to(lowercase			)
    model.eval()
    __UpperCamelCase				      =  model(
        lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	start_positions=lowercase		,	end_positions=lowercase		,	)
    self.parent.assertEqual(result.start_logits.shape		,	(self.batch_size, self.seq_length)			)
    self.parent.assertEqual(result.end_logits.shape		,	(self.batch_size, self.seq_length)			)
  
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    __UpperCamelCase				      =  self.prepare_config_and_inputs()
    (
        (
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		(
        __UpperCamelCase
    )      ,		
    )				      =  config_and_inputs
    __UpperCamelCase				      =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
    return config, inputs_dict
  
  
  @require_torch
  class        UpperCAmelCase__  (    UpperCAmelCase_							,			UpperCAmelCase_							,			unittest.TestCase):
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				(
       (
           FunnelModel,
           FunnelForMaskedLM,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   __SCREAMING_SNAKE_CASE					      =				(
       {
           '''feature-extraction''': (FunnelBaseModel, FunnelModel),
           '''fill-mask''': FunnelForMaskedLM,
           '''question-answering''': FunnelForQuestionAnswering,
           '''text-classification''': FunnelForSequenceClassification,
           '''token-classification''': FunnelForTokenClassification,
           '''zero-shot''': FunnelForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase=False			)  ->							Tuple:
    __UpperCamelCase				      =  super()._prepare_for_class(lowercase		,	lowercase		,	return_labels=lowercase			)
  
    if return_labels:
     if model_class in get_values(lowercase			):
      __UpperCamelCase				      =  torch.zeros(
          (self.model_tester.batch_size, self.model_tester.seq_length)		,	dtype=torch.long		,	device=lowercase			)
    return inputs_dict
  
  
  
   def    __lowerCamelCase				(  self			)  ->							List[str]:
    __UpperCamelCase				      =  FunnelModelTester(self			)
    __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    self.config_tester.run_common_tests()
  
  
  
   def    __lowerCamelCase				(  self			)  ->							List[str]:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_pretraining(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							str:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Dict:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*lowercase			)
  
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							int:
    if hasattr(lowercase		,	"""weight"""			) and module.weight is not None:
     module.weight.data.fill_(3			)
    if hasattr(lowercase		,	"""bias"""			) and module.bias is not None:
     module.bias.data.fill_(3			)
  
    for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
     if hasattr(lowercase		,	lowercase			) and getattr(lowercase		,	lowercase			) is not None:
      __UpperCamelCase				      =  getattr(lowercase		,	lowercase			)
      weight.data.fill_(3			)
  
  
  @require_torch
  class        UpperCAmelCase__  (    UpperCAmelCase_							,			unittest.TestCase):
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				(
       (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
   )
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  FunnelModelTester(self		,	base=lowercase			)
    __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    self.config_tester.run_common_tests()
  
  
  
   def    __lowerCamelCase				(  self			)  ->							str:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_base_model(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Any:
    __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*lowercase			)
  
  
  
   def    __lowerCamelCase				(  self			)  ->							str:
    __UpperCamelCase      ,		__UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs_for_common()
    __UpperCamelCase				      =  True
  
    for model_class in self.all_model_classes:
     if model_class.__name__ == "FunnelBaseModel":
      continue
     __UpperCamelCase				      =  model_class(lowercase			)
     model.to(lowercase			)
     model.train()
     __UpperCamelCase				      =  self._prepare_for_class(lowercase		,	lowercase		,	return_labels=lowercase			)
     __UpperCamelCase				      =  model(**lowercase			).loss
     loss.backward()
  
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[Any]:
    if hasattr(lowercase		,	"""weight"""			) and module.weight is not None:
     module.weight.data.fill_(3			)
    if hasattr(lowercase		,	"""bias"""			) and module.bias is not None:
     module.bias.data.fill_(3			)
  
    for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
     if hasattr(lowercase		,	lowercase			) and getattr(lowercase		,	lowercase			) is not None:
      __UpperCamelCase				      =  getattr(lowercase		,	lowercase			)
      weight.data.fill_(3			)
  
  
  @require_torch
  @require_sentencepiece
  @require_tokenizers
  class        UpperCAmelCase__  (    unittest.TestCase):
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    __UpperCamelCase				      =  1_3
    __UpperCamelCase				      =  7
    __UpperCamelCase				      =  torch.arange(0		,	batch_size * sequence_length			).long().reshape(lowercase		,	lowercase			)
    __UpperCamelCase				      =  [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
    __UpperCamelCase				      =  torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]			)
  
    __UpperCamelCase				      =  FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""			)
    __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)[0].abs()
  
    __UpperCamelCase				      =  torch.tensor(2_344.8_352			)
    __UpperCamelCase				      =  torch.tensor(0.8_052			)
    self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
    self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)
  
    __UpperCamelCase				      =  torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]			)
    __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)[0].abs()
  
    __UpperCamelCase				      =  torch.tensor(2_343.8_425			)
    __UpperCamelCase				      =  torch.tensor(0.8_049			)
    self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
    self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)
  
  
  
  
   @slow
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""			)
    __UpperCamelCase				      =  FunnelModel.from_pretrained("""huggingface/funnel-small"""			)
    __UpperCamelCase				      =  tokenizer("""Hello! I am the Funnel Transformer model."""		,	return_tensors="""pt"""			)
    __UpperCamelCase				      =  model(**lowercase			)[0]
  
    __UpperCamelCase				      =  torch.tensor(235.7_246			)
    __UpperCamelCase				      =  torch.tensor(0.0_256			)
    self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
    self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)
  
  
  
  
  
  