



'''simple docstring'''



import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings('ignore')

a__						:			Optional[int]										=			"""K-MEANS-CLUST/ """







def _lowercase   (  __A							,__A							,__A=None       ):



 '''simple docstring'''



 if seed is not None:  # useful for obtaining consistent results
  np.random.seed(__A       )
 __UpperCamelCase				      =  data.shape[0]  # number of data points

 # Pick K indices from range [0, N).
 __UpperCamelCase				      =  np.random.randint(0							,__A							,__A       )

 # Keep centroids as dense format, as many entries will be nonzero due to averaging.
 # As long as at least one document in a cluster contains a word,
 # it will carry a nonzero weight in the TF-IDF vector of the centroid.
 __UpperCamelCase				      =  data[rand_indices, :]

 return centroids







def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 return pairwise_distances(__A							,__A							,metric="""euclidean"""       )







def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 __UpperCamelCase				      =  centroid_pairwise_dist(__A							,__A       )

 # Compute cluster assignments for each data point:
 # Fill in the blank (RHS only)
 __UpperCamelCase				      =  np.argmin(__A							,axis=1       )

 return cluster_assignment







def _lowercase   (  __A							,__A							,__A       ):



 '''simple docstring'''



 __UpperCamelCase				      =  []
 for i in range(__A       ):
  # Select all data points that belong to cluster i. Fill in the blank (RHS only)
  __UpperCamelCase				      =  data[cluster_assignment == i]
  # Compute the mean of the data points. Fill in the blank (RHS only)
  __UpperCamelCase				      =  member_data_points.mean(axis=0       )
  new_centroids.append(__A       )
 __UpperCamelCase				      =  np.array(__A       )

 return new_centroids







def _lowercase   (  __A							,__A							,__A							,__A       ):



 '''simple docstring'''



 __UpperCamelCase				      =  0.0
 for i in range(__A       ):
  # Select all data points that belong to cluster i. Fill in the blank (RHS only)
  __UpperCamelCase				      =  data[cluster_assignment == i, :]

  if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
   # Compute distances from centroid to data points (RHS only)
   __UpperCamelCase				      =  pairwise_distances(
       __A							,[centroids[i]]							,metric="""euclidean"""       )
   __UpperCamelCase				      =  distances**2
   heterogeneity += np.sum(__A       )

 return heterogeneity







def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 plt.figure(figsize=(7, 4)       )
 plt.plot(__A							,linewidth=4       )
 plt.xlabel("""# Iterations"""       )
 plt.ylabel("""Heterogeneity"""       )
 plt.title(f"Heterogeneity of clustering over time, K={k:d}"       )
 plt.rcParams.update({"""font.size""": 16}       )
 plt.show()







def _lowercase   (  __A							,__A							,__A							,__A=500							,__A=None							,__A=False       ):



 '''simple docstring'''



 __UpperCamelCase				      =  initial_centroids[:]
 __UpperCamelCase				      =  None

 for itr in range(__A       ):
  if verbose:
   print(__A							,end=""""""       )

  # 1. Make cluster assignments using nearest centroids
  __UpperCamelCase				      =  assign_clusters(__A							,__A       )

  # 2. Compute a new centroid for each of the k clusters, averaging all data
  #    points assigned to that cluster.
  __UpperCamelCase				      =  revise_centroids(__A							,__A							,__A       )

  # Check for convergence: if none of the assignments changed, stop
  if (
      prev_cluster_assignment is not None
      and (prev_cluster_assignment == cluster_assignment).all()
  ):
   break

  # Print number of new assignments
  if prev_cluster_assignment is not None:
   __UpperCamelCase				      =  np.sum(prev_cluster_assignment != cluster_assignment       )
   if verbose:
    print(
        f"    {num_changed:5d} elements changed their cluster assignment."       )

        # Record heterogeneity convergence metric
  if record_heterogeneity is not None:
   # YOUR CODE HERE
   __UpperCamelCase				      =  compute_heterogeneity(__A							,__A							,__A							,__A       )
   record_heterogeneity.append(__A       )

  __UpperCamelCase				      =  cluster_assignment[:]

 return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
 from sklearn import datasets as ds

 a__						:			Optional[Any]										=			ds.load_iris()
 a__						:			Optional[Any]										=			3
 a__						:			int										=			[]
 a__						:			int										=			get_initial_centroids(dataset['data'], k, seed=0)
 a__						:			Any										=			kmeans(
     dataset['data'],
     k,
     initial_centroids,
     maxiter=4_0_0,
     record_heterogeneity=heterogeneity,
     verbose=True,
 )
 plot_heterogeneity(heterogeneity, k)







def _lowercase   (  __A							,__A							,__A=None       ):



 '''simple docstring'''



 if fill_missing_report:
  __UpperCamelCase				      =  df.fillna(value=__A       )
 __UpperCamelCase				      =  1
 __UpperCamelCase				      =  df.select_dtypes(np.number       ).columns
 __UpperCamelCase				      =  (
     df.groupby(["""Cluster"""]       )[  # construct report dataframe
         numeric_cols
     ]  # group by cluster number
     .agg(
         [
             ("""sum""", np.sum),
             ("""mean_with_zeros""", lambda __A       : np.mean(np.nan_to_num(__A       )       )),
             ("""mean_without_zeros""", lambda __A       : x.replace(0							,np.NaN       ).mean()),
             (
                 """mean_25-75""",
                 lambda __A       : np.mean(
                     np.nan_to_num(
                         sorted(__A       )[
                             round(len(__A       ) * 25 / 100       ) : round(len(__A       ) * 75 / 100       )
                         ]       )       ),
             ),
             ("""mean_with_na""", np.mean),
             ("""min""", lambda __A       : x.min()),
             ("""5%""", lambda __A       : x.quantile(0.05       )),
             ("""25%""", lambda __A       : x.quantile(0.25       )),
             ("""50%""", lambda __A       : x.quantile(0.50       )),
             ("""75%""", lambda __A       : x.quantile(0.75       )),
             ("""95%""", lambda __A       : x.quantile(0.95       )),
             ("""max""", lambda __A       : x.max()),
             ("""count""", lambda __A       : x.count()),
             ("""stdev""", lambda __A       : x.std()),
             ("""mode""", lambda __A       : x.mode()[0]),
             ("""median""", lambda __A       : x.median()),
             ("""# > 0""", lambda __A       : (x > 0).sum()),
         ]       )
     .T.reset_index()
     .rename(index=__A							,columns={"""level_0""": """Features""", """level_1""": """Type"""}       )
 )  # rename columns
 # calculate the size of cluster(count of clientID's)
 __UpperCamelCase				      =  report[
     (report["""Features"""] == """dummy""") & (report["""Type"""] == """count""")
 ].copy()  # avoid SettingWithCopyWarning
 __UpperCamelCase				      =  (
     """ClusterSize"""  # rename created cluster df to match report column names
 )
 __UpperCamelCase				      =  """# of Customers"""
 __UpperCamelCase				      =  pd.DataFrame(
     clustersize.iloc[:, 2:].values
     / clustersize.iloc[:, 2:].values.sum()       )
 __UpperCamelCase				      =  """% of Customers"""  # rename created cluster df to match report column names
 __UpperCamelCase				      =  """ClusterProportion"""
 __UpperCamelCase				      =  clusterproportion.columns.tolist()
 __UpperCamelCase				      =  cols[-2:] + cols[:-2]
 __UpperCamelCase				      =  clusterproportion[cols]  # rearrange columns to match report
 __UpperCamelCase				      =  report.columns
 __UpperCamelCase				      =  pd.DataFrame(
     abs(
         report[report["""Type"""] == """count"""].iloc[:, 2:].values
         - clustersize.iloc[:, 2:].values       )       )  # generating df with count of nan values
 __UpperCamelCase				      =  0
 __UpperCamelCase				      =  """# of nan"""
 __UpperCamelCase				      =  report[
     report["""Type"""] == """count"""
 ].Features.tolist()  # filling values in order to match report
 __UpperCamelCase				      =  a.columns.tolist()
 __UpperCamelCase				      =  cols[-2:] + cols[:-2]
 __UpperCamelCase				      =  a[cols]  # rearrange columns to match report
 __UpperCamelCase				      =  report.columns  # rename columns to match report
 __UpperCamelCase				      =  report.drop(
     report[report.Type == """count"""].index       )  # drop count values except cluster size
 __UpperCamelCase				      =  pd.concat(
     [report, a, clustersize, clusterproportion]							,axis=0       )  # concat report with clustert size and nan values
 __UpperCamelCase				      =  report["""Features"""].isin(__A       )
 __UpperCamelCase				      =  report.columns.tolist()
 __UpperCamelCase				      =  cols[0:2] + cols[-1:] + cols[2:-1]
 __UpperCamelCase				      =  report[cols]
 __UpperCamelCase				      =  {
     """ClusterSize""": 9,
     """ClusterProportion""": 8,
     """mean_with_zeros""": 7,
     """mean_with_na""": 6,
     """max""": 5,
     """50%""": 4,
     """min""": 3,
     """25%""": 2,
     """75%""": 1,
     """# of nan""": 0,
     """# > 0""": -1,
     """sum_with_na""": -2,
 }
 __UpperCamelCase				      =  (
     report.assign(
         Sortera=lambda __A       : x.Type.map(__A       )							,Sortera=lambda __A       : list(reversed(range(len(__A       )       )       )       )							,)
     .sort_values(["""Sorter1""", """Mark""", """Sorter2"""]							,ascending=__A       )
     .drop(["""Sorter1""", """Sorter2"""]							,axis=1       )
 )
 __UpperCamelCase				      =  """"""
 __UpperCamelCase				      =  report.reset_index()
 __UpperCamelCase				      =  report.drop(columns=["""index"""]       )
 return report


if __name__ == "__main__":
 import doctest

 doctest.testmod()










'''simple docstring'''



import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     FunnelBaseModel,
     FunnelForMaskedLM,
     FunnelForMultipleChoice,
     FunnelForPreTraining,
     FunnelForQuestionAnswering,
     FunnelForSequenceClassification,
     FunnelForTokenClassification,
     FunnelModel,
 )
class        UpperCAmelCase__  :



 def __init__(  self		,	lowercase		,	lowercase=1_3		,	lowercase=7		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=9_9		,	lowercase=[1, 1, 2]		,	lowercase=1		,	lowercase=3_2		,	lowercase=4		,	lowercase=8		,	lowercase=3_7		,	lowercase="gelu_new"		,	lowercase=0.1		,	lowercase=0.1		,	lowercase=0.0		,	lowercase=5_1_2		,	lowercase=3		,	lowercase=0.02		,	lowercase=3		,	lowercase=4		,	lowercase=None		,	lowercase=False		,	)  ->							Any:
  __UpperCamelCase				      =  parent
  __UpperCamelCase				      =  batch_size
  __UpperCamelCase				      =  seq_length
  __UpperCamelCase				      =  is_training
  __UpperCamelCase				      =  use_input_mask
  __UpperCamelCase				      =  use_token_type_ids
  __UpperCamelCase				      =  use_labels
  __UpperCamelCase				      =  vocab_size
  __UpperCamelCase				      =  block_sizes
  __UpperCamelCase				      =  num_decoder_layers
  __UpperCamelCase				      =  d_model
  __UpperCamelCase				      =  n_head
  __UpperCamelCase				      =  d_head
  __UpperCamelCase				      =  d_inner
  __UpperCamelCase				      =  hidden_act
  __UpperCamelCase				      =  hidden_dropout
  __UpperCamelCase				      =  attention_dropout
  __UpperCamelCase				      =  activation_dropout
  __UpperCamelCase				      =  max_position_embeddings
  __UpperCamelCase				      =  type_vocab_size
  __UpperCamelCase				      =  2
  __UpperCamelCase				      =  num_labels
  __UpperCamelCase				      =  num_choices
  __UpperCamelCase				      =  scope
  __UpperCamelCase				      =  initializer_std

  # Used in the tests to check the size of the first attention layer
  __UpperCamelCase				      =  n_head
  # Used in the tests to check the size of the first hidden state
  __UpperCamelCase				      =  self.d_model
  # Used in the tests to check the number of output hidden states/attentions
  __UpperCamelCase				      =  sum(self.block_sizes			) + (0 if base else self.num_decoder_layers)
  # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
  # the last hidden state of the first block (which is the first hidden state of the decoder).
  if not base:
   __UpperCamelCase				      =  self.num_hidden_layers + 2



 def    __lowerCamelCase				(  self			)  ->							Tuple:
  __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size			)

  __UpperCamelCase				      =  None
  if self.use_input_mask:
   __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	vocab_size=2			)

  __UpperCamelCase				      =  None
  if self.use_token_type_ids:
   __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size			)

  __UpperCamelCase				      =  None
  __UpperCamelCase				      =  None
  __UpperCamelCase				      =  None
  if self.use_labels:
   __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.type_sequence_label_size			)
   __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.num_labels			)
   __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.num_choices			)
   __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	1			)

  __UpperCamelCase				      =  self.get_config()

  return (
      config,
      input_ids,
      token_type_ids,
      input_mask,
      sequence_labels,
      token_labels,
      choice_labels,
      fake_token_labels,
  )



 def    __lowerCamelCase				(  self			)  ->							Any:
  return FunnelConfig(
      vocab_size=self.vocab_size		,	block_sizes=self.block_sizes		,	num_decoder_layers=self.num_decoder_layers		,	d_model=self.d_model		,	n_head=self.n_head		,	d_head=self.d_head		,	d_inner=self.d_inner		,	hidden_act=self.hidden_act		,	hidden_dropout=self.hidden_dropout		,	attention_dropout=self.attention_dropout		,	activation_dropout=self.activation_dropout		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	initializer_std=self.initializer_std		,	)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
  __UpperCamelCase				      =  FunnelModel(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)
  __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)

  __UpperCamelCase				      =  False
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)

  __UpperCamelCase				      =  False
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.d_model)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							str:
  __UpperCamelCase				      =  FunnelBaseModel(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)
  __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)			)

  __UpperCamelCase				      =  False
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 3, self.d_model)			)

  __UpperCamelCase				      =  False
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, 2, self.d_model)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
  __UpperCamelCase				      =  self.num_labels
  __UpperCamelCase				      =  FunnelForPreTraining(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
  __UpperCamelCase				      =  FunnelForMaskedLM(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.vocab_size)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
  __UpperCamelCase				      =  self.num_labels
  __UpperCamelCase				      =  FunnelForSequenceClassification(lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_labels)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							int:
  __UpperCamelCase				      =  self.num_choices
  __UpperCamelCase				      =  FunnelForMultipleChoice(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  input_ids.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
  __UpperCamelCase				      =  token_type_ids.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
  __UpperCamelCase				      =  input_mask.unsqueeze(1			).expand(-1		,	self.num_choices		,	-1			).contiguous()
  __UpperCamelCase				      =  model(
      lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase		,	)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_choices)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Optional[Any]:
  __UpperCamelCase				      =  self.num_labels
  __UpperCamelCase				      =  FunnelForTokenClassification(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	labels=lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.num_labels)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	)  ->							Any:
  __UpperCamelCase				      =  FunnelForQuestionAnswering(config=lowercase			)
  model.to(lowercase			)
  model.eval()
  __UpperCamelCase				      =  model(
      lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase		,	start_positions=lowercase		,	end_positions=lowercase		,	)
  self.parent.assertEqual(result.start_logits.shape		,	(self.batch_size, self.seq_length)			)
  self.parent.assertEqual(result.end_logits.shape		,	(self.batch_size, self.seq_length)			)




 def    __lowerCamelCase				(  self			)  ->							Any:
  __UpperCamelCase				      =  self.prepare_config_and_inputs()
  (
      (
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		(
      __UpperCamelCase
  )      ,		
  )				      =  config_and_inputs
  __UpperCamelCase				      =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
  return config, inputs_dict


@require_torch
class        UpperCAmelCase__  (    UpperCAmelCase_							,			UpperCAmelCase_							,			unittest.TestCase):
 __SCREAMING_SNAKE_CASE					      =				False
 __SCREAMING_SNAKE_CASE					      =				False
 __SCREAMING_SNAKE_CASE					      =				(
     (
         FunnelModel,
         FunnelForMaskedLM,
         FunnelForPreTraining,
         FunnelForQuestionAnswering,
         FunnelForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 __SCREAMING_SNAKE_CASE					      =				(
     {
         '''feature-extraction''': (FunnelBaseModel, FunnelModel),
         '''fill-mask''': FunnelForMaskedLM,
         '''question-answering''': FunnelForQuestionAnswering,
         '''text-classification''': FunnelForSequenceClassification,
         '''token-classification''': FunnelForTokenClassification,
         '''zero-shot''': FunnelForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase=False			)  ->							Tuple:
  __UpperCamelCase				      =  super()._prepare_for_class(lowercase		,	lowercase		,	return_labels=lowercase			)

  if return_labels:
   if model_class in get_values(lowercase			):
    __UpperCamelCase				      =  torch.zeros(
        (self.model_tester.batch_size, self.model_tester.seq_length)		,	dtype=torch.long		,	device=lowercase			)
  return inputs_dict



 def    __lowerCamelCase				(  self			)  ->							List[str]:
  __UpperCamelCase				      =  FunnelModelTester(self			)
  __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Tuple:
  self.config_tester.run_common_tests()



 def    __lowerCamelCase				(  self			)  ->							List[str]:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_model(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Optional[int]:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_pretraining(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							str:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_masked_lm(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Dict:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_token_classification(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Tuple:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_question_answering(*lowercase			)




 def    __lowerCamelCase				(  self		,	lowercase			)  ->							int:
  if hasattr(lowercase		,	"""weight"""			) and module.weight is not None:
   module.weight.data.fill_(3			)
  if hasattr(lowercase		,	"""bias"""			) and module.bias is not None:
   module.bias.data.fill_(3			)

  for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
   if hasattr(lowercase		,	lowercase			) and getattr(lowercase		,	lowercase			) is not None:
    __UpperCamelCase				      =  getattr(lowercase		,	lowercase			)
    weight.data.fill_(3			)


@require_torch
class        UpperCAmelCase__  (    UpperCAmelCase_							,			unittest.TestCase):
 __SCREAMING_SNAKE_CASE					      =				False
 __SCREAMING_SNAKE_CASE					      =				False
 __SCREAMING_SNAKE_CASE					      =				(
     (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
 )



 def    __lowerCamelCase				(  self			)  ->							Optional[int]:
  __UpperCamelCase				      =  FunnelModelTester(self		,	base=lowercase			)
  __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Any:
  self.config_tester.run_common_tests()



 def    __lowerCamelCase				(  self			)  ->							str:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_base_model(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Any:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_sequence_classification(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Any:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_multiple_choice(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							str:
  __UpperCamelCase      ,		__UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs_for_common()
  __UpperCamelCase				      =  True

  for model_class in self.all_model_classes:
   if model_class.__name__ == "FunnelBaseModel":
    continue
   __UpperCamelCase				      =  model_class(lowercase			)
   model.to(lowercase			)
   model.train()
   __UpperCamelCase				      =  self._prepare_for_class(lowercase		,	lowercase		,	return_labels=lowercase			)
   __UpperCamelCase				      =  model(**lowercase			).loss
   loss.backward()




 def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[Any]:
  if hasattr(lowercase		,	"""weight"""			) and module.weight is not None:
   module.weight.data.fill_(3			)
  if hasattr(lowercase		,	"""bias"""			) and module.bias is not None:
   module.bias.data.fill_(3			)

  for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
   if hasattr(lowercase		,	lowercase			) and getattr(lowercase		,	lowercase			) is not None:
    __UpperCamelCase				      =  getattr(lowercase		,	lowercase			)
    weight.data.fill_(3			)


@require_torch
@require_sentencepiece
@require_tokenizers
class        UpperCAmelCase__  (    unittest.TestCase):



 def    __lowerCamelCase				(  self			)  ->							Tuple:
  __UpperCamelCase				      =  1_3
  __UpperCamelCase				      =  7
  __UpperCamelCase				      =  torch.arange(0		,	batch_size * sequence_length			).long().reshape(lowercase		,	lowercase			)
  __UpperCamelCase				      =  [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
  __UpperCamelCase				      =  torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]			)

  __UpperCamelCase				      =  FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""			)
  __UpperCamelCase				      =  model(lowercase		,	token_type_ids=lowercase			)[0].abs()

  __UpperCamelCase				      =  torch.tensor(2_344.8_352			)
  __UpperCamelCase				      =  torch.tensor(0.8_052			)
  self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
  self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)

  __UpperCamelCase				      =  torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]			)
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase		,	token_type_ids=lowercase			)[0].abs()

  __UpperCamelCase				      =  torch.tensor(2_343.8_425			)
  __UpperCamelCase				      =  torch.tensor(0.8_049			)
  self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
  self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)




 @slow
 def    __lowerCamelCase				(  self			)  ->							Optional[int]:
  __UpperCamelCase				      =  FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""			)
  __UpperCamelCase				      =  FunnelModel.from_pretrained("""huggingface/funnel-small"""			)
  __UpperCamelCase				      =  tokenizer("""Hello! I am the Funnel Transformer model."""		,	return_tensors="""pt"""			)
  __UpperCamelCase				      =  model(**lowercase			)[0]

  __UpperCamelCase				      =  torch.tensor(235.7_246			)
  __UpperCamelCase				      =  torch.tensor(0.0_256			)
  self.assertTrue(torch.allclose(output.sum()		,	lowercase		,	atol=1E-4			)			)
  self.assertTrue(torch.allclose(output.mean()		,	lowercase		,	atol=1E-4			)			)





