

"""simple docstring"""





import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				List[Any]   ,lowercase_			:				Tuple   ,lowercase_			:				Dict   ,lowercase_			:				str		):
						lowerCAmelCase__	:						int  				=	dataset
						lowerCAmelCase__	:						List[str]  				=	process
						lowerCAmelCase__	:						Dict  				=	params







					def __len__(       self			:				Any		):
						return len(self.dataset		)







					def __getitem__(       self			:				Union[str, Any]   ,lowercase_			:				List[Any]		):
						lowerCAmelCase__	:						Union[str, Any]  				=	self.dataset[i]
						lowerCAmelCase__	:						Optional[Any]  				=	self.process(lowercase_   ,**self.params		)
						return processed







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				Optional[int]   ,lowercase_			:				Optional[Any]   ,lowercase_			:				List[Any]   ,lowercase_			:				Optional[Any]   ,lowercase_			:				Tuple=None		):
						lowerCAmelCase__	:						List[Any]  				=	loader
						lowerCAmelCase__	:						int  				=	infer
						lowerCAmelCase__	:						List[str]  				=	params
						if loader_batch_size == 1:
							# Let's spare some time by deactivating altogether
							lowerCAmelCase__	:						int  				=	None
						lowerCAmelCase__	:						Dict  				=	loader_batch_size

						# Internal bookkeeping
						lowerCAmelCase__	:						Optional[Any]  				=	None
						lowerCAmelCase__	:						Optional[int]  				=	None







					def __len__(       self			:				Union[str, Any]		):
						return len(self.loader		)







					def __iter__(       self			:				List[Any]		):
						lowerCAmelCase__	:						List[Any]  				=	iter(self.loader		)
						return self







					def        __lowerCAmelCase (       self			:				Tuple		):
						if isinstance(self._loader_batch_data   ,torch.Tensor		):
							# Batch data is simple tensor, just fetch the slice
							lowerCAmelCase__	:						Tuple  				=	self._loader_batch_data[self._loader_batch_index]
						else:
							# Batch data is assumed to be BaseModelOutput (or dict)
							lowerCAmelCase__	:						int  				=	{}
							for k, element in self._loader_batch_data.items():
								if isinstance(lowercase_   ,lowercase_		):
									# Convert ModelOutput to tuple first
									lowerCAmelCase__	:						List[Any]  				=	element.to_tuple()
									if isinstance(element[0]   ,torch.Tensor		):
										lowerCAmelCase__	:						List[Any]  				=	tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
									elif isinstance(element[0]   ,np.ndarray		):
										lowerCAmelCase__	:						str  				=	tuple(np.expand_dims(el[self._loader_batch_index]   ,0		) for el in element		)
									continue
								if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(lowercase_   ,lowercase_		):
									# Those are stored as lists of tensors so need specific unbatching.
									if isinstance(element[0]   ,torch.Tensor		):
										lowerCAmelCase__	:						Dict  				=	tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
									elif isinstance(element[0]   ,np.ndarray		):
										lowerCAmelCase__	:						Optional[int]  				=	tuple(np.expand_dims(el[self._loader_batch_index]   ,0		) for el in element		)
									continue
								if element is None:
									# This can happen for optional data that get passed around
									lowerCAmelCase__	:						Dict  				=	None
								elif isinstance(element[self._loader_batch_index]   ,torch.Tensor		):
									# Take correct batch data, but make it looked like batch_size=1
									# For compatibility with other methods within transformers

									lowerCAmelCase__	:						str  				=	element[self._loader_batch_index].unsqueeze(0		)
								elif isinstance(element[self._loader_batch_index]   ,np.ndarray		):
									# Take correct batch data, but make it looked like batch_size=1
									# For compatibility with other methods within transformers
									lowerCAmelCase__	:						Tuple  				=	np.expand_dims(element[self._loader_batch_index]   ,0		)
								else:
									# This is typically a list, so no need to `unsqueeze`.
									lowerCAmelCase__	:						int  				=	element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
							lowerCAmelCase__	:						int  				=	self._loader_batch_data.__class__(lowercase_		)
						self._loader_batch_index += 1
						return result







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
							# We are currently unrolling a batch so we just need to return
							# the current item within a batch
							return self.loader_batch_item()

						# We're out of items within a batch
						lowerCAmelCase__	:						Dict  				=	next(self.iterator		)
						lowerCAmelCase__	:						List[Any]  				=	self.infer(lowercase_   ,**self.params		)
						# We now have a batch of "inferred things".
						if self.loader_batch_size is not None:
							# Try to infer the size of the batch
							if isinstance(lowercase_   ,torch.Tensor		):
								lowerCAmelCase__	:						int  				=	processed
							else:
								lowerCAmelCase__	:						Union[str, Any]  				=	list(processed.keys()		)[0]
								lowerCAmelCase__	:						Union[str, Any]  				=	processed[key]
							if isinstance(lowercase_   ,lowercase_		):
								lowerCAmelCase__	:						List[Any]  				=	len(lowercase_		)
							else:
								lowerCAmelCase__	:						List[str]  				=	first_tensor.shape[0]
							if 0 < observed_batch_size < self.loader_batch_size:
								# could be last batch so we can't unroll as many
								# elements.
								lowerCAmelCase__	:						Optional[Any]  				=	observed_batch_size
							# Setting internal index to unwrap the batch
							lowerCAmelCase__	:						str  				=	processed
							lowerCAmelCase__	:						Any  				=	0
							return self.loader_batch_item()
						else:
							# We're not unrolling batches
							return processed







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				int   ,lowercase_			:				str   ,lowercase_			:				str   ,lowercase_			:				Union[str, Any]   ,lowercase_			:				int=None		):
						super().__init__(lowercase_   ,lowercase_   ,lowercase_		)







					def __iter__(       self			:				List[Any]		):
						lowerCAmelCase__	:						Dict  				=	iter(self.loader		)
						lowerCAmelCase__	:						Tuple  				=	None
						return self







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						if self.subiterator is None:
							lowerCAmelCase__	:						List[Any]  				=	self.infer(next(self.iterator		)   ,**self.params		)
						try:
							# Try to return next item
							lowerCAmelCase__	:						Optional[int]  				=	next(self.subiterator		)
						except StopIteration:
							# When a preprocess iterator ends, we can start lookig at the next item
							# ChunkIterator will keep feeding until ALL elements of iterator
							# all have created their subiterator and have been iterating against.
							#
							# Another way to look at it, is we're basically flattening lists of lists
							# into a single list, but with generators
							lowerCAmelCase__	:						Any  				=	self.infer(next(self.iterator		)   ,**self.params		)
							lowerCAmelCase__	:						int  				=	next(self.subiterator		)
						return processed







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __iter__(       self			:				Tuple		):
						lowerCAmelCase__	:						int  				=	iter(self.loader		)
						return self







					def        __lowerCAmelCase (       self			:				List[Any]		):
						# Extremely similar to PipelineIterator in its unpacking mechanism
						# BUT, we have an extra required item which is the presence of `is_last`
						# That is because everything is flattened by `PipelineChunkIterator` we
						# need to keep track of how to regroup here in the original `process`
						# boundaries so that `process` and `postprocess` see the same data.

						# This iterator accumulates items (possibly while unbatching) until it
						# its a `is_last` and then just passes it on to the caller.
						lowerCAmelCase__	:						Tuple  				=	False
						lowerCAmelCase__	:						str  				=	[]
						if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
							while self._loader_batch_index < self.loader_batch_size:
								lowerCAmelCase__	:						Dict  				=	self.loader_batch_item()
								lowerCAmelCase__	:						Optional[Any]  				=	item.pop('''is_last'''		)
								accumulator.append(lowercase_		)
								if is_last:
									return accumulator

						while not is_last:
							lowerCAmelCase__	:						Any  				=	self.infer(next(self.iterator		)   ,**self.params		)
							if self.loader_batch_size is not None:
								if isinstance(lowercase_   ,torch.Tensor		):
									lowerCAmelCase__	:						Tuple  				=	processed
								else:
									lowerCAmelCase__	:						List[Any]  				=	list(processed.keys()		)[0]
									lowerCAmelCase__	:						Union[str, Any]  				=	processed[key]
								if isinstance(lowercase_   ,lowercase_		):
									lowerCAmelCase__	:						Tuple  				=	len(lowercase_		)
								else:
									lowerCAmelCase__	:						str  				=	first_tensor.shape[0]
								if 0 < observed_batch_size < self.loader_batch_size:
									# could be last batch so we can't unroll as many
									# elements.
									lowerCAmelCase__	:						Optional[int]  				=	observed_batch_size
								lowerCAmelCase__	:						Optional[int]  				=	processed
								lowerCAmelCase__	:						Optional[int]  				=	0
								while self._loader_batch_index < self.loader_batch_size:
									lowerCAmelCase__	:						Any  				=	self.loader_batch_item()
									lowerCAmelCase__	:						Optional[Any]  				=	item.pop('''is_last'''		)
									accumulator.append(lowercase_		)
									if is_last:
										return accumulator
							else:
								lowerCAmelCase__	:						Dict  				=	processed
								lowerCAmelCase__	:						Tuple  				=	item.pop('''is_last'''		)
								accumulator.append(lowercase_		)
						return accumulator







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				int   ,lowercase_			:				Dataset   ,lowercase_			:				str		):
						lowerCAmelCase__	:						List[Any]  				=	dataset
						lowerCAmelCase__	:						List[Any]  				=	key







					def __len__(       self			:				List[Any]		):
						return len(self.dataset		)







					def __getitem__(       self			:				str   ,lowercase_			:				Union[str, Any]		):
						return self.dataset[i][self.key]







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				Dict   ,lowercase_			:				Dataset   ,lowercase_			:				str   ,lowercase_			:				str		):
						lowerCAmelCase__	:						str  				=	dataset
						lowerCAmelCase__	:						List[str]  				=	keya
						lowerCAmelCase__	:						Optional[Any]  				=	keya







					def __len__(       self			:				str		):
						return len(self.dataset		)







					def __getitem__(       self			:				Optional[int]   ,lowercase_			:				Union[str, Any]		):
						return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}







"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  RoCBertTokenizer
				SCREAMING_SNAKE_CASE_   :      Any         =  None
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  True
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  filter_non_english

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						super().setUp()

						__lowerCAmelCase:       Optional[Any]          =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
						__lowerCAmelCase:       List[Any]          =			{}
						__lowerCAmelCase:       Dict          =			{}
						for i, value in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       List[Any]          =			i
								__lowerCAmelCase:       Union[str, Any]          =			i
						__lowerCAmelCase:       List[str]          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"])
						with open(self.vocab_file     ,  "w"     ,  encoding="utf-8") as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
						with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8") as word_shape_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)
						with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8") as word_pronunciation_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.tokenize("你好[SEP]你是谁")
						self.assertListEqual(UpperCamelCase__     ,  ["你", "好", "[SEP]", "你", "是", "谁"])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")     ,  ["ah", "\u535A", "\u63A8", "zz"])

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["hello", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hällo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["h\u00E9llo"])

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				str)->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[Any])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Tuple)->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  never_split=["[UNK]"])

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

				def 							lowercase_     (	self							:				List[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__lowerCAmelCase:       int          =			{}
						for i, token in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       Optional[Any]          =			i
						__lowerCAmelCase:       str          =			RoCBertWordpieceTokenizer(vocab=UpperCamelCase__     ,  unk_token="[UNK]")

						self.assertListEqual(tokenizer.tokenize("")     ,  [])

						self.assertListEqual(tokenizer.tokenize("unwanted running")     ,  ["un", "##want", "##ed", "runn", "##ing"])

						self.assertListEqual(tokenizer.tokenize("unwantedX running")     ,  ["[UNK]", "runn", "##ing"])

				def 							lowercase_     (	self							:				Optional[Any])->   Dict:



						'''simple docstring'''


						self.assertTrue(_is_whitespace(" "))
						self.assertTrue(_is_whitespace("\t"))
						self.assertTrue(_is_whitespace("\r"))
						self.assertTrue(_is_whitespace("\n"))
						self.assertTrue(_is_whitespace("\u00A0"))

						self.assertFalse(_is_whitespace("A"))
						self.assertFalse(_is_whitespace("-"))

				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						self.assertTrue(_is_control("\u0005"))

						self.assertFalse(_is_control("A"))
						self.assertFalse(_is_control(" "))
						self.assertFalse(_is_control("\t"))
						self.assertFalse(_is_control("\r"))

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						self.assertTrue(_is_punctuation("-"))
						self.assertTrue(_is_punctuation("$"))
						self.assertTrue(_is_punctuation("`"))
						self.assertTrue(_is_punctuation("."))

						self.assertFalse(_is_punctuation("A"))
						self.assertFalse(_is_punctuation(" "))

				def 							lowercase_     (	self							:				Dict)->   int:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

						if self.test_rust_tokenizer:
								__lowerCAmelCase:       Any          =			self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       str          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowerCAmelCase:       Tuple          =			tokenizer_r.encode_plus(
										    UpperCamelCase__     ,  return_attention_mask=UpperCamelCase__     ,  return_token_type_ids=UpperCamelCase__     ,  return_offsets_mapping=UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  )

										__lowerCAmelCase:       str          =			tokenizer_r.do_lower_case if hasattr(UpperCamelCase__     ,  "do_lower_case") else False
										__lowerCAmelCase:       List[Any]          =			(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "Allen"),
										        ((2_1, 2_3), "##NL"),
										        ((2_3, 2_4), "##P"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "allen"),
										        ((2_1, 2_3), "##nl"),
										        ((2_3, 2_4), "##p"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"])

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			["的", "人", "有"]
						__lowerCAmelCase:       int          =			"".join(UpperCamelCase__)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Tuple          =			True
										__lowerCAmelCase:       str          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Union[str, Any]          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       List[Any]          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Any          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       List[str]          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

										__lowerCAmelCase:       int          =			False
										__lowerCAmelCase:       Any          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       str          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase:       Dict          =			[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCamelCase__)
										]
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				@slow
				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Dict          =			tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase__)
						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase__)

						__lowerCAmelCase:       Tuple          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]




				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.get_tokenizers(do_lower_case=UpperCamelCase__)
						for tokenizer in tokenizers:
								with self.subTest(f"{tokenizer.__class__.__name__}"):
										__lowerCAmelCase:       str          =			"你好，你是谁"
										__lowerCAmelCase:       Dict          =			tokenizer.tokenize(UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			tokenizer.convert_tokens_to_ids(UpperCamelCase__)
										__lowerCAmelCase:       Optional[Any]          =			tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			tokenizer.prepare_for_model(
										    UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			tokenizer.encode_plus(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)


