






"""simple docstring"""

import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings("""ignore""")

lowercase__      =				"""K-MEANS-CLUST/ """

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase=None )		->					Optional[Any]:

		"""simple docstring"""







		if seed is not None:  # useful for obtaining consistent results
				np.random.seed(__UpperCamelCase )
		lowerCAmelCase_     :							List[Any]             =    data.shape[0]  # number of data points

		# Pick K indices from range [0, N).
		lowerCAmelCase_     :							Optional[Any]             =    np.random.randint(0   , __UpperCamelCase   , __UpperCamelCase )

		# Keep centroids as dense format, as many entries will be nonzero due to averaging.
		# As long as at least one document in a cluster contains a word,
		# it will carry a nonzero weight in the TF-IDF vector of the centroid.
		lowerCAmelCase_     :							List[str]             =    data[rand_indices, :]

		return centroids

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase )		->					Dict:

		"""simple docstring"""







		return pairwise_distances(__UpperCamelCase   , __UpperCamelCase   , metric="euclidean" )

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase )		->					List[str]:

		"""simple docstring"""







		lowerCAmelCase_     :							Optional[Any]             =    centroid_pairwise_dist(__UpperCamelCase   , __UpperCamelCase )

		# Compute cluster assignments for each data point:
		# Fill in the blank (RHS only)
		lowerCAmelCase_     :							str             =    np.argmin(__UpperCamelCase   , axis=1 )

		return cluster_assignment

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )		->					Dict:

		"""simple docstring"""







		lowerCAmelCase_     :							Tuple             =    []
		for i in range(__UpperCamelCase ):
				# Select all data points that belong to cluster i. Fill in the blank (RHS only)
				lowerCAmelCase_     :							List[Any]             =    data[cluster_assignment == i]
				# Compute the mean of the data points. Fill in the blank (RHS only)
				lowerCAmelCase_     :							List[str]             =    member_data_points.mean(axis=0 )
				new_centroids.append(__UpperCamelCase )
		lowerCAmelCase_     :							Tuple             =    np.array(__UpperCamelCase )

		return new_centroids

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )		->					int:

		"""simple docstring"""







		lowerCAmelCase_     :							Union[str, Any]             =    0.0
		for i in range(__UpperCamelCase ):
				# Select all data points that belong to cluster i. Fill in the blank (RHS only)
				lowerCAmelCase_     :							List[str]             =    data[cluster_assignment == i, :]

				if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
						# Compute distances from centroid to data points (RHS only)
						lowerCAmelCase_     :							int             =    pairwise_distances(
						    __UpperCamelCase   , [centroids[i]]   , metric="euclidean" )
						lowerCAmelCase_     :							Optional[int]             =    distances**2
						heterogeneity += np.sum(__UpperCamelCase )

		return heterogeneity

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase )		->					Dict:

		"""simple docstring"""







		plt.figure(figsize=(7, 4) )
		plt.plot(__UpperCamelCase   , linewidth=4 )
		plt.xlabel("# Iterations" )
		plt.ylabel("Heterogeneity" )
		plt.title(f'''Heterogeneity of clustering over time, K={k:d}''' )
		plt.rcParams.update({"font.size": 16} )
		plt.show()

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase=500   , __UpperCamelCase=None   , __UpperCamelCase=False )		->					List[Any]:

		"""simple docstring"""







		lowerCAmelCase_     :							Dict             =    initial_centroids[:]
		lowerCAmelCase_     :							Optional[int]             =    None

		for itr in range(__UpperCamelCase ):
				if verbose:
						print(__UpperCamelCase   , end="" )

				# 1. Make cluster assignments using nearest centroids
				lowerCAmelCase_     :							Optional[Any]             =    assign_clusters(__UpperCamelCase   , __UpperCamelCase )

				# 2. Compute a new centroid for each of the k clusters, averaging all data
				#    points assigned to that cluster.
				lowerCAmelCase_     :							Tuple             =    revise_centroids(__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )

				# Check for convergence: if none of the assignments changed, stop
				if (
				    prev_cluster_assignment is not None
				    and (prev_cluster_assignment == cluster_assignment).all()
				):
						break

				# Print number of new assignments
				if prev_cluster_assignment is not None:
						lowerCAmelCase_     :							Union[str, Any]             =    np.sum(prev_cluster_assignment != cluster_assignment )
						if verbose:
								print(
								    f'''    {num_changed:5d} elements changed their cluster assignment.''' )

        # Record heterogeneity convergence metric
				if record_heterogeneity is not None:
						# YOUR CODE HERE
						lowerCAmelCase_     :							Tuple             =    compute_heterogeneity(__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase )
						record_heterogeneity.append(__UpperCamelCase )

				lowerCAmelCase_     :							str             =    cluster_assignment[:]

		return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
					from sklearn import datasets as ds

					lowercase__      =				ds.load_iris()
					lowercase__      =				3
					lowercase__      =				[]
					lowercase__      =				get_initial_centroids(dataset["""data"""], k, seed=0)
					lowercase__ ,     lowercase__      =				kmeans(
					    dataset["""data"""],
					    k,
					    initial_centroids,
					    maxiter=400,
					    record_heterogeneity=heterogeneity,
					    verbose=True,
					)
					plot_heterogeneity(heterogeneity, k)

def 	__lowerCamelCase			(						__UpperCamelCase   , __UpperCamelCase   , __UpperCamelCase=None )		->					pd.DataFrame:

		"""simple docstring"""







		if fill_missing_report:
				lowerCAmelCase_     :							str             =    df.fillna(value=__UpperCamelCase )
		lowerCAmelCase_     :							Optional[int]             =    1
		lowerCAmelCase_     :							Optional[int]             =    df.select_dtypes(np.number ).columns
		lowerCAmelCase_     :							Dict             =    (
		    df.groupby(["Cluster"] )[  # construct report dataframe
		        numeric_cols
		    ]  # group by cluster number
		    .agg(
		        [
		            ("sum", np.sum),
		            ("mean_with_zeros", lambda __UpperCamelCase : np.mean(np.nan_to_num(__UpperCamelCase ) )),
		            ("mean_without_zeros", lambda __UpperCamelCase : x.replace(0   , np.NaN ).mean()),
		            (
		                "mean_25-75",
		                lambda __UpperCamelCase : np.mean(
		                    np.nan_to_num(
		                        sorted(__UpperCamelCase )[
		                            round(len(__UpperCamelCase ) * 25 / 100 ) : round(len(__UpperCamelCase ) * 75 / 100 )
		                        ] ) ),
		            ),
		            ("mean_with_na", np.mean),
		            ("min", lambda __UpperCamelCase : x.min()),
		            ("5%", lambda __UpperCamelCase : x.quantile(0.05 )),
		            ("25%", lambda __UpperCamelCase : x.quantile(0.25 )),
		            ("50%", lambda __UpperCamelCase : x.quantile(0.50 )),
		            ("75%", lambda __UpperCamelCase : x.quantile(0.75 )),
		            ("95%", lambda __UpperCamelCase : x.quantile(0.95 )),
		            ("max", lambda __UpperCamelCase : x.max()),
		            ("count", lambda __UpperCamelCase : x.count()),
		            ("stdev", lambda __UpperCamelCase : x.std()),
		            ("mode", lambda __UpperCamelCase : x.mode()[0]),
		            ("median", lambda __UpperCamelCase : x.median()),
		            ("# > 0", lambda __UpperCamelCase : (x > 0).sum()),
		        ] )
		    .T.reset_index()
		    .rename(index=__UpperCamelCase   , columns={"level_0": "Features", "level_1": "Type"} )
		)  # rename columns
		# calculate the size of cluster(count of clientID's)
		lowerCAmelCase_     :							Optional[Any]             =    report[
		    (report["Features"] == "dummy") & (report["Type"] == "count")
		].copy()  # avoid SettingWithCopyWarning
		lowerCAmelCase_     :							List[Any]             =    (
		    "ClusterSize"  # rename created cluster df to match report column names
		)
		lowerCAmelCase_     :							Dict             =    "# of Customers"
		lowerCAmelCase_     :							List[str]             =    pd.DataFrame(
		    clustersize.iloc[:, 2:].values
		    / clustersize.iloc[:, 2:].values.sum() )
		lowerCAmelCase_     :							Optional[Any]             =    "% of Customers"  # rename created cluster df to match report column names
		lowerCAmelCase_     :							Dict             =    "ClusterProportion"
		lowerCAmelCase_     :							Optional[Any]             =    clusterproportion.columns.tolist()
		lowerCAmelCase_     :							Optional[Any]             =    cols[-2:] + cols[:-2]
		lowerCAmelCase_     :							Dict             =    clusterproportion[cols]  # rearrange columns to match report
		lowerCAmelCase_     :							Union[str, Any]             =    report.columns
		lowerCAmelCase_     :							Dict             =    pd.DataFrame(
		    abs(
		        report[report["Type"] == "count"].iloc[:, 2:].values
		        - clustersize.iloc[:, 2:].values ) )  # generating df with count of nan values
		lowerCAmelCase_     :							Union[str, Any]             =    0
		lowerCAmelCase_     :							Any             =    "# of nan"
		lowerCAmelCase_     :							Dict             =    report[
		    report["Type"] == "count"
		].Features.tolist()  # filling values in order to match report
		lowerCAmelCase_     :							Optional[Any]             =    a.columns.tolist()
		lowerCAmelCase_     :							Optional[Any]             =    cols[-2:] + cols[:-2]
		lowerCAmelCase_     :							str             =    a[cols]  # rearrange columns to match report
		lowerCAmelCase_     :							Tuple             =    report.columns  # rename columns to match report
		lowerCAmelCase_     :							List[str]             =    report.drop(
		    report[report.Type == "count"].index )  # drop count values except cluster size
		lowerCAmelCase_     :							Tuple             =    pd.concat(
		    [report, a, clustersize, clusterproportion]   , axis=0 )  # concat report with clustert size and nan values
		lowerCAmelCase_     :							List[Any]             =    report["Features"].isin(__UpperCamelCase )
		lowerCAmelCase_     :							Tuple             =    report.columns.tolist()
		lowerCAmelCase_     :							List[str]             =    cols[0:2] + cols[-1:] + cols[2:-1]
		lowerCAmelCase_     :							List[str]             =    report[cols]
		lowerCAmelCase_     :							str             =    {
		    "ClusterSize": 9,
		    "ClusterProportion": 8,
		    "mean_with_zeros": 7,
		    "mean_with_na": 6,
		    "max": 5,
		    "50%": 4,
		    "min": 3,
		    "25%": 2,
		    "75%": 1,
		    "# of nan": 0,
		    "# > 0": -1,
		    "sum_with_na": -2,
		}
		lowerCAmelCase_     :							List[str]             =    (
		    report.assign(
		        Sortera=lambda __UpperCamelCase : x.Type.map(__UpperCamelCase )   , Sortera=lambda __UpperCamelCase : list(reversed(range(len(__UpperCamelCase ) ) ) )   , )
		    .sort_values(["Sorter1", "Mark", "Sorter2"]   , ascending=__UpperCamelCase )
		    .drop(["Sorter1", "Sorter2"]   , axis=1 )
		)
		lowerCAmelCase_     :							str             =    ""
		lowerCAmelCase_     :							Any             =    report.reset_index()
		lowerCAmelCase_     :							Any             =    report.drop(columns=["index"] )
		return report


if __name__ == "__main__":
					import doctest

					doctest.testmod()














"""simple docstring"""

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 	__lowerCamelCase       (				A__				,							unittest.TestCase    ):





		'''simple docstring'''






		a_			:       List[str]      =							RealmTokenizer
		a_			:       int      =							RealmTokenizerFast
		a_			:       str      =							True
		a_			:       Tuple      =							True
		a_			:       List[Any]      =							filter_non_english





		def      lowerCamelCase (     self							:			int			):
				super().setUp()

				lowerCAmelCase_     :							int             =    [
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				lowerCAmelCase_     :							List[Any]             =    os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["vocab_file"]			)
				with open(self.vocab_file		,    "w"		,    encoding="utf-8"			) as vocab_writer:
						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)





		def      lowerCamelCase (     self							:			Optional[int]		,    a_							:			Any			):
				lowerCAmelCase_     :							int             =    "UNwant\u00E9d,running"
				lowerCAmelCase_     :							Optional[Any]             =    "unwanted, running"
				return input_text, output_text





		def      lowerCamelCase (     self							:			int			):
				lowerCAmelCase_     :							Tuple             =    self.tokenizer_class(self.vocab_file			)

				lowerCAmelCase_     :							List[str]             =    tokenizer.tokenize("UNwant\u00E9d,running"			)
				self.assertListEqual(a_		,    ["un", "##want", "##ed", ",", "runn", "##ing"]			)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)		,    [9, 6, 7, 12, 10, 11]			)





		def      lowerCamelCase (     self							:			Dict			):
				if not self.test_rust_tokenizer:
						return

				lowerCAmelCase_     :							str             =    self.get_tokenizer()
				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer()

				lowerCAmelCase_     :							Optional[Any]             =    "UNwant\u00E9d,running"

				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.tokenize(a_			)
				lowerCAmelCase_     :							List[Any]             =    rust_tokenizer.tokenize(a_			)
				self.assertListEqual(a_		,    a_			)

				lowerCAmelCase_     :							str             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
				self.assertListEqual(a_		,    a_			)

				lowerCAmelCase_     :							int             =    self.get_rust_tokenizer()
				lowerCAmelCase_     :							Any             =    tokenizer.encode(a_			)
				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.encode(a_			)
				self.assertListEqual(a_		,    a_			)

				# With lower casing
				lowerCAmelCase_     :							Dict             =    self.get_tokenizer(do_lower_case=a_			)
				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer(do_lower_case=a_			)

				lowerCAmelCase_     :							Dict             =    "UNwant\u00E9d,running"

				lowerCAmelCase_     :							Union[str, Any]             =    tokenizer.tokenize(a_			)
				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.tokenize(a_			)
				self.assertListEqual(a_		,    a_			)

				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
				self.assertListEqual(a_		,    a_			)

				lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
				lowerCAmelCase_     :							str             =    tokenizer.encode(a_			)
				lowerCAmelCase_     :							Optional[int]             =    rust_tokenizer.encode(a_			)
				self.assertListEqual(a_		,    a_			)





		def      lowerCamelCase (     self							:			Optional[int]			):
				lowerCAmelCase_     :							Any             =    BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)		,    ["ah", "\u535A", "\u63A8", "zz"]			)





		def      lowerCamelCase (     self							:			Tuple			):
				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["hello", "!", "how", "are", "you", "?"]			)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)





		def      lowerCamelCase (     self							:			Optional[Any]			):
				lowerCAmelCase_     :							Optional[int]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hällo", "!", "how", "are", "you", "?"]			)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["h\u00E9llo"]			)





		def      lowerCamelCase (     self							:			Tuple			):
				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)





		def      lowerCamelCase (     self							:			str			):
				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)





		def      lowerCamelCase (     self							:			List[Any]			):
				lowerCAmelCase_     :							Optional[Any]             =    BasicTokenizer(do_lower_case=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?"]			)





		def      lowerCamelCase (     self							:			List[Any]			):
				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HäLLo", "!", "how", "Are", "yoU", "?"]			)





		def      lowerCamelCase (     self							:			Dict			):
				lowerCAmelCase_     :							List[str]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HaLLo", "!", "how", "Are", "yoU", "?"]			)





		def      lowerCamelCase (     self							:			List[str]			):
				lowerCAmelCase_     :							Any             =    BasicTokenizer(do_lower_case=a_		,    never_split=["[UNK]"]			)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)





		def      lowerCamelCase (     self							:			Optional[Any]			):
				lowerCAmelCase_     :							int             =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				lowerCAmelCase_     :							Union[str, Any]             =    {}
				for i, token in enumerate(a_			):
						lowerCAmelCase_     :							Any             =    i
				lowerCAmelCase_     :							int             =    WordpieceTokenizer(vocab=a_		,    unk_token="[UNK]"			)

				self.assertListEqual(tokenizer.tokenize(""			)		,    []			)

				self.assertListEqual(tokenizer.tokenize("unwanted running"			)		,    ["un", "##want", "##ed", "runn", "##ing"]			)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"			)		,    ["[UNK]", "runn", "##ing"]			)





		def      lowerCamelCase (     self							:			Tuple			):
				self.assertTrue(_is_whitespace(" "			)			)
				self.assertTrue(_is_whitespace("\t"			)			)
				self.assertTrue(_is_whitespace("\r"			)			)
				self.assertTrue(_is_whitespace("\n"			)			)
				self.assertTrue(_is_whitespace("\u00A0"			)			)

				self.assertFalse(_is_whitespace("A"			)			)
				self.assertFalse(_is_whitespace("-"			)			)





		def      lowerCamelCase (     self							:			Optional[Any]			):
				self.assertTrue(_is_control("\u0005"			)			)

				self.assertFalse(_is_control("A"			)			)
				self.assertFalse(_is_control(" "			)			)
				self.assertFalse(_is_control("\t"			)			)
				self.assertFalse(_is_control("\r"			)			)





		def      lowerCamelCase (     self							:			str			):
				self.assertTrue(_is_punctuation("-"			)			)
				self.assertTrue(_is_punctuation("$"			)			)
				self.assertTrue(_is_punctuation("`"			)			)
				self.assertTrue(_is_punctuation("."			)			)

				self.assertFalse(_is_punctuation("A"			)			)
				self.assertFalse(_is_punctuation(" "			)			)





		def      lowerCamelCase (     self							:			str			):
				lowerCAmelCase_     :							Any             =    self.get_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)

				if self.test_rust_tokenizer:
						lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
						self.assertListEqual(
						    [rust_tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)





		@slow
		def      lowerCamelCase (     self							:			int			):
				lowerCAmelCase_     :							int             =    self.tokenizer_class.from_pretrained("bert-base-uncased"			)

				lowerCAmelCase_     :							List[Any]             =    tokenizer.encode("sequence builders"		,    add_special_tokens=a_			)
				lowerCAmelCase_     :							Optional[int]             =    tokenizer.encode("multi-sequence build"		,    add_special_tokens=a_			)

				lowerCAmelCase_     :							Dict             =    tokenizer.build_inputs_with_special_tokens(a_			)
				lowerCAmelCase_     :							Tuple             =    tokenizer.build_inputs_with_special_tokens(a_		,    a_			)

				assert encoded_sentence == [1_01] + text + [1_02]
				assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]





		def      lowerCamelCase (     self							:			Optional[int]			):
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)

								lowerCAmelCase_     :							Tuple             =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
								lowerCAmelCase_     :							List[str]             =    tokenizer_r.encode_plus(
								    a_		,    return_attention_mask=a_		,    return_token_type_ids=a_		,    return_offsets_mapping=a_		,    add_special_tokens=a_		,    )

								lowerCAmelCase_     :							List[str]             =    tokenizer_r.do_lower_case if hasattr(a_		,    "do_lower_case"			) else False
								lowerCAmelCase_     :							int             =    (
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), "A"),
								        ((1, 2), ","),
								        ((3, 5), "na"),
								        ((5, 6), "##ï"),
								        ((6, 8), "##ve"),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), "Allen"),
								        ((21, 23), "##NL"),
								        ((23, 24), "##P"),
								        ((25, 33), "sentence"),
								        ((33, 34), "."),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), "a"),
								        ((1, 2), ","),
								        ((3, 8), "naive"),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), "allen"),
								        ((21, 23), "##nl"),
								        ((23, 24), "##p"),
								        ((25, 33), "sentence"),
								        ((33, 34), "."),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
								self.assertEqual([e[0] for e in expected_results]		,    tokens["offset_mapping"]			)





		@slow
		def      lowerCamelCase (     self							:			int			):
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
								lowerCAmelCase_     :							Tuple             =    self.tokenizer_class.from_pretrained(a_		,    **a_			)
								lowerCAmelCase_     :							Dict             =    [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

								lowerCAmelCase_     :							Optional[Any]             =    tokenizer_r.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
								lowerCAmelCase_     :							int             =    tokenizer_p.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)

								lowerCAmelCase_     :							Dict             =    (2, 2, 10)

								self.assertEqual(encoded_sentence_r["input_ids"].shape		,    a_			)
								self.assertEqual(encoded_sentence_r["attention_mask"].shape		,    a_			)
								self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,    a_			)

								self.assertEqual(encoded_sentence_p["input_ids"].shape		,    a_			)
								self.assertEqual(encoded_sentence_p["attention_mask"].shape		,    a_			)
								self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,    a_			)
