code: 
  
  
  
  
  
  
  """simple docstring"""
  
  import warnings
  
  import numpy as np
  import pandas as pd
  from matplotlib import pyplot as plt
  from sklearn.metrics import pairwise_distances
  
  warnings.filterwarnings("""ignore""")
  
  lowercase__      =				"""K-MEANS-CLUST/ """
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase=None )		->					Optional[Any]:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		if seed is not None:  # useful for obtaining consistent results
  				np.random.seed(__UpperCamelCase )
  		lowerCAmelCase_     :							List[Any]             =    data.shape[0]  # number of data points
  
  		# Pick K indices from range [0, N).
  		lowerCAmelCase_     :							Optional[Any]             =    np.random.randint(0   , __UpperCamelCase   , __UpperCamelCase )
  
  		# Keep centroids as dense format, as many entries will be nonzero due to averaging.
  		# As long as at least one document in a cluster contains a word,
  		# it will carry a nonzero weight in the TF-IDF vector of the centroid.
  		lowerCAmelCase_     :							List[str]             =    data[rand_indices, :]
  
  		return centroids
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase )		->					Dict:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		return pairwise_distances(__UpperCamelCase   , __UpperCamelCase   , metric="euclidean" )
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase )		->					List[str]:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							Optional[Any]             =    centroid_pairwise_dist(__UpperCamelCase   , __UpperCamelCase )
  
  		# Compute cluster assignments for each data point:
  		# Fill in the blank (RHS only)
  		lowerCAmelCase_     :							str             =    np.argmin(__UpperCamelCase   , axis=1 )
  
  		return cluster_assignment
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )		->					Dict:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							Tuple             =    []
  		for i in range(__UpperCamelCase ):
  				# Select all data points that belong to cluster i. Fill in the blank (RHS only)
  				lowerCAmelCase_     :							List[Any]             =    data[cluster_assignment == i]
  				# Compute the mean of the data points. Fill in the blank (RHS only)
  				lowerCAmelCase_     :							List[str]             =    member_data_points.mean(axis=0 )
  				new_centroids.append(__UpperCamelCase )
  		lowerCAmelCase_     :							Tuple             =    np.array(__UpperCamelCase )
  
  		return new_centroids
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )		->					int:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							Union[str, Any]             =    0.0
  		for i in range(__UpperCamelCase ):
  				# Select all data points that belong to cluster i. Fill in the blank (RHS only)
  				lowerCAmelCase_     :							List[str]             =    data[cluster_assignment == i, :]
  
  				if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
  						# Compute distances from centroid to data points (RHS only)
  						lowerCAmelCase_     :							int             =    pairwise_distances(
  						    __UpperCamelCase   , [centroids[i]]   , metric="euclidean" )
  						lowerCAmelCase_     :							Optional[int]             =    distances**2
  						heterogeneity += np.sum(__UpperCamelCase )
  
  		return heterogeneity
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase )		->					Dict:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		plt.figure(figsize=(7, 4) )
  		plt.plot(__UpperCamelCase   , linewidth=4 )
  		plt.xlabel("# Iterations" )
  		plt.ylabel("Heterogeneity" )
  		plt.title(f'''Heterogeneity of clustering over time, K={k:d}''' )
  		plt.rcParams.update({"font.size": 16} )
  		plt.show()
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase=500   , __UpperCamelCase=None   , __UpperCamelCase=False )		->					List[Any]:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		lowerCAmelCase_     :							Dict             =    initial_centroids[:]
  		lowerCAmelCase_     :							Optional[int]             =    None
  
  		for itr in range(__UpperCamelCase ):
  				if verbose:
  						print(__UpperCamelCase   , end="" )
  
  				# 1. Make cluster assignments using nearest centroids
  				lowerCAmelCase_     :							Optional[Any]             =    assign_clusters(__UpperCamelCase   , __UpperCamelCase )
  
  				# 2. Compute a new centroid for each of the k clusters, averaging all data
  				#    points assigned to that cluster.
  				lowerCAmelCase_     :							Tuple             =    revise_centroids(__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )
  
  				# Check for convergence: if none of the assignments changed, stop
  				if (
  				    prev_cluster_assignment is not None
  				    and (prev_cluster_assignment == cluster_assignment).all()
  				):
  						break
  
  				# Print number of new assignments
  				if prev_cluster_assignment is not None:
  						lowerCAmelCase_     :							Union[str, Any]             =    np.sum(prev_cluster_assignment != cluster_assignment )
  						if verbose:
  								print(
  								    f'''    {num_changed:5d} elements changed their cluster assignment.''' )
  
          # Record heterogeneity convergence metric
  				if record_heterogeneity is not None:
  						# YOUR CODE HERE
  						lowerCAmelCase_     :							Tuple             =    compute_heterogeneity(__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )
  						record_heterogeneity.append(__UpperCamelCase )
  
  				lowerCAmelCase_     :							str             =    cluster_assignment[:]
  
  		return centroids, cluster_assignment
  
  
  # Mock test below
  if False:  # change to true to run this test case.
  					from sklearn import datasets as ds
  
  					lowercase__      =				ds.load_iris()
  					lowercase__      =				3
  					lowercase__      =				[]
  					lowercase__      =				get_initial_centroids(dataset["""data"""], k, seed=0)
  					lowercase__ ,     lowercase__      =				kmeans(
  					    dataset["""data"""],
  					    k,
  					    initial_centroids,
  					    maxiter=400,
  					    record_heterogeneity=heterogeneity,
  					    verbose=True,
  					)
  					plot_heterogeneity(heterogeneity, k)
  
  def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase=None )		->					pd.DataFrame:
  
  		"""simple docstring"""
  
  
  
  
  
  
  
  		if fill_missing_report:
  				lowerCAmelCase_     :							str             =    df.fillna(value=__UpperCamelCase )
  		lowerCAmelCase_     :							Optional[int]             =    1
  		lowerCAmelCase_     :							Optional[int]             =    df.select_dtypes(np.number ).columns
  		lowerCAmelCase_     :							Dict             =    (
  		    df.groupby(["Cluster"] )[  # construct report dataframe
  		        numeric_cols
  		    ]  # group by cluster number
  		    .agg(
  		        [
  		            ("sum", np.sum),
  		            ("mean_with_zeros", lambda __UpperCamelCase : np.mean(np.nan_to_num(__UpperCamelCase ) )),
  		            ("mean_without_zeros", lambda __UpperCamelCase : x.replace(0   , np.NaN ).mean()),
  		            (
  		                "mean_25-75",
  		                lambda __UpperCamelCase : np.mean(
  		                    np.nan_to_num(
  		                        sorted(__UpperCamelCase )[
  		                            round(len(__UpperCamelCase ) * 25 / 100 ) : round(len(__UpperCamelCase ) * 75 / 100 )
  		                        ] ) ),
  		            ),
  		            ("mean_with_na", np.mean),
  		            ("min", lambda __UpperCamelCase : x.min()),
  		            ("5%", lambda __UpperCamelCase : x.quantile(0.05 )),
  		            ("25%", lambda __UpperCamelCase : x.quantile(0.25 )),
  		            ("50%", lambda __UpperCamelCase : x.quantile(0.50 )),
  		            ("75%", lambda __UpperCamelCase : x.quantile(0.75 )),
  		            ("95%", lambda __UpperCamelCase : x.quantile(0.95 )),
  		            ("max", lambda __UpperCamelCase : x.max()),
  		            ("count", lambda __UpperCamelCase : x.count()),
  		            ("stdev", lambda __UpperCamelCase : x.std()),
  		            ("mode", lambda __UpperCamelCase : x.mode()[0]),
  		            ("median", lambda __UpperCamelCase : x.median()),
  		            ("# > 0", lambda __UpperCamelCase : (x > 0).sum()),
  		        ] )
  		    .T.reset_index()
  		    .rename(index=__UpperCamelCase   , columns={"level_0": "Features", "level_1": "Type"} )
  		)  # rename columns
  		# calculate the size of cluster(count of clientID's)
  		lowerCAmelCase_     :							Optional[Any]             =    report[
  		    (report["Features"] == "dummy") & (report["Type"] == "count")
  		].copy()  # avoid SettingWithCopyWarning
  		lowerCAmelCase_     :							List[Any]             =    (
  		    "ClusterSize"  # rename created cluster df to match report column names
  		)
  		lowerCAmelCase_     :							Dict             =    "# of Customers"
  		lowerCAmelCase_     :							List[str]             =    pd.DataFrame(
  		    clustersize.iloc[:, 2:].values
  		    / clustersize.iloc[:, 2:].values.sum() )
  		lowerCAmelCase_     :							Optional[Any]             =    "% of Customers"  # rename created cluster df to match report column names
  		lowerCAmelCase_     :							Dict             =    "ClusterProportion"
  		lowerCAmelCase_     :							Optional[Any]             =    clusterproportion.columns.tolist()
  		lowerCAmelCase_     :							Optional[Any]             =    cols[-2:] + cols[:-2]
  		lowerCAmelCase_     :							Dict             =    clusterproportion[cols]  # rearrange columns to match report
  		lowerCAmelCase_     :							Union[str, Any]             =    report.columns
  		lowerCAmelCase_     :							Dict             =    pd.DataFrame(
  		    abs(
  		        report[report["Type"] == "count"].iloc[:, 2:].values
  		        - clustersize.iloc[:, 2:].values ) )  # generating df with count of nan values
  		lowerCAmelCase_     :							Union[str, Any]             =    0
  		lowerCAmelCase_     :							Any             =    "# of nan"
  		lowerCAmelCase_     :							Dict             =    report[
  		    report["Type"] == "count"
  		].Features.tolist()  # filling values in order to match report
  		lowerCAmelCase_     :							Optional[Any]             =    a.columns.tolist()
  		lowerCAmelCase_     :							Optional[Any]             =    cols[-2:] + cols[:-2]
  		lowerCAmelCase_     :							str             =    a[cols]  # rearrange columns to match report
  		lowerCAmelCase_     :							Tuple             =    report.columns  # rename columns to match report
  		lowerCAmelCase_     :							List[str]             =    report.drop(
  		    report[report.Type == "count"].index )  # drop count values except cluster size
  		lowerCAmelCase_     :							Tuple             =    pd.concat(
  		    [report, a, clustersize, clusterproportion]   , axis=0 )  # concat report with clustert size and nan values
  		lowerCAmelCase_     :							List[Any]             =    report["Features"].isin(__UpperCamelCase )
  		lowerCAmelCase_     :							Tuple             =    report.columns.tolist()
  		lowerCAmelCase_     :							List[str]             =    cols[0:2] + cols[-1:] + cols[2:-1]
  		lowerCAmelCase_     :							List[str]             =    report[cols]
  		lowerCAmelCase_     :							str             =    {
  		    "ClusterSize": 9,
  		    "ClusterProportion": 8,
  		    "mean_with_zeros": 7,
  		    "mean_with_na": 6,
  		    "max": 5,
  		    "50%": 4,
  		    "min": 3,
  		    "25%": 2,
  		    "75%": 1,
  		    "# of nan": 0,
  		    "# > 0": -1,
  		    "sum_with_na": -2,
  		}
  		lowerCAmelCase_     :							List[str]             =    (
  		    report.assign(
  		        Sortera=lambda __UpperCamelCase : x.Type.map(__UpperCamelCase )   , Sortera=lambda __UpperCamelCase : list(reversed(range(len(__UpperCamelCase ) ) ) )   , )
  		    .sort_values(["Sorter1", "Mark", "Sorter2"]   , ascending=__UpperCamelCase )
  		    .drop(["Sorter1", "Sorter2"]   , axis=1 )
  		)
  		lowerCAmelCase_     :							str             =    ""
  		lowerCAmelCase_     :							Any             =    report.reset_index()
  		lowerCAmelCase_     :							Any             =    report.drop(columns=["index"] )
  		return report
  
  
  if __name__ == "__main__":
  					import doctest
  
  					doctest.testmod()
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  """simple docstring"""
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class 	__lowerCamelCase       (				A__				,							unittest.TestCase    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		a_			:       List[str]      =							RealmTokenizer
  		a_			:       int      =							RealmTokenizerFast
  		a_			:       str      =							True
  		a_			:       Tuple      =							True
  		a_			:       List[Any]      =							filter_non_english
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				super().setUp()
  
  				lowerCAmelCase_     :							int             =    [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				lowerCAmelCase_     :							List[Any]             =    os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["vocab_file"]			)
  				with open(self.vocab_file		,    "w"		,    encoding="utf-8"			) as vocab_writer:
  						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]		,    a_							:			Any			):
  				lowerCAmelCase_     :							int             =    "UNwant\u00E9d,running"
  				lowerCAmelCase_     :							Optional[Any]             =    "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							Tuple             =    self.tokenizer_class(self.vocab_file			)
  
  				lowerCAmelCase_     :							List[str]             =    tokenizer.tokenize("UNwant\u00E9d,running"			)
  				self.assertListEqual(a_		,    ["un", "##want", "##ed", ",", "runn", "##ing"]			)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)		,    [9, 6, 7, 12, 10, 11]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				if not self.test_rust_tokenizer:
  						return
  
  				lowerCAmelCase_     :							str             =    self.get_tokenizer()
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer()
  
  				lowerCAmelCase_     :							Optional[Any]             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							List[Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							int             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							Any             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				# With lower casing
  				lowerCAmelCase_     :							Dict             =    self.get_tokenizer(do_lower_case=a_			)
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer(do_lower_case=a_			)
  
  				lowerCAmelCase_     :							Dict             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Union[str, Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Optional[int]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)		,    ["ah", "\u535A", "\u63A8", "zz"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["hello", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							Optional[int]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hällo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["h\u00E9llo"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							Optional[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HäLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				lowerCAmelCase_     :							List[str]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HaLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[str]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer(do_lower_case=a_		,    never_split=["[UNK]"]			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							int             =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				lowerCAmelCase_     :							Union[str, Any]             =    {}
  				for i, token in enumerate(a_			):
  						lowerCAmelCase_     :							Any             =    i
  				lowerCAmelCase_     :							int             =    WordpieceTokenizer(vocab=a_		,    unk_token="[UNK]"			)
  
  				self.assertListEqual(tokenizer.tokenize(""			)		,    []			)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"			)		,    ["un", "##want", "##ed", "runn", "##ing"]			)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"			)		,    ["[UNK]", "runn", "##ing"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				self.assertTrue(_is_whitespace(" "			)			)
  				self.assertTrue(_is_whitespace("\t"			)			)
  				self.assertTrue(_is_whitespace("\r"			)			)
  				self.assertTrue(_is_whitespace("\n"			)			)
  				self.assertTrue(_is_whitespace("\u00A0"			)			)
  
  				self.assertFalse(_is_whitespace("A"			)			)
  				self.assertFalse(_is_whitespace("-"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				self.assertTrue(_is_control("\u0005"			)			)
  
  				self.assertFalse(_is_control("A"			)			)
  				self.assertFalse(_is_control(" "			)			)
  				self.assertFalse(_is_control("\t"			)			)
  				self.assertFalse(_is_control("\r"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				self.assertTrue(_is_punctuation("-"			)			)
  				self.assertTrue(_is_punctuation("$"			)			)
  				self.assertTrue(_is_punctuation("`"			)			)
  				self.assertTrue(_is_punctuation("."			)			)
  
  				self.assertFalse(_is_punctuation("A"			)			)
  				self.assertFalse(_is_punctuation(" "			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							Any             =    self.get_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  				if self.test_rust_tokenizer:
  						lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  						self.assertListEqual(
  						    [rust_tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							int             =    self.tokenizer_class.from_pretrained("bert-base-uncased"			)
  
  				lowerCAmelCase_     :							List[Any]             =    tokenizer.encode("sequence builders"		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Optional[int]             =    tokenizer.encode("multi-sequence build"		,    add_special_tokens=a_			)
  
  				lowerCAmelCase_     :							Dict             =    tokenizer.build_inputs_with_special_tokens(a_			)
  				lowerCAmelCase_     :							Tuple             =    tokenizer.build_inputs_with_special_tokens(a_		,    a_			)
  
  				assert encoded_sentence == [1_01] + text + [1_02]
  				assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  
  								lowerCAmelCase_     :							Tuple             =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.encode_plus(
  								    a_		,    return_attention_mask=a_		,    return_token_type_ids=a_		,    return_offsets_mapping=a_		,    add_special_tokens=a_		,    )
  
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.do_lower_case if hasattr(a_		,    "do_lower_case"			) else False
  								lowerCAmelCase_     :							int             =    (
  								    [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "A"),
  								        ((1, 2), ","),
  								        ((3, 5), "na"),
  								        ((5, 6), "##ï"),
  								        ((6, 8), "##ve"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "Allen"),
  								        ((21, 23), "##NL"),
  								        ((23, 24), "##P"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								    if not do_lower_case
  								    else [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "a"),
  								        ((1, 2), ","),
  								        ((3, 8), "naive"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "allen"),
  								        ((21, 23), "##nl"),
  								        ((23, 24), "##p"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								)
  
  								self.assertEqual(
  								    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
  								self.assertEqual([e[0] for e in expected_results]		,    tokens["offset_mapping"]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Tuple             =    self.tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Dict             =    [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]
  
  								lowerCAmelCase_     :							Optional[Any]             =    tokenizer_r.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  								lowerCAmelCase_     :							int             =    tokenizer_p.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  
  								lowerCAmelCase_     :							Dict             =    (2, 2, 10)
  
  								self.assertEqual(encoded_sentence_r["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,    a_			)
  
  								self.assertEqual(encoded_sentence_p["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,    a_			)
  