

"""simple docstring"""


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__A						   =			datasets.utils.logging.get_logger(__name__)





@dataclass
class     snake_case	(				datasets.BuilderConfig  ):
				SCREAMING_SNAKE_CASE_   :      Optional[datasets.Features]         =  None
				SCREAMING_SNAKE_CASE_   :      str         =  "utf-8"
				SCREAMING_SNAKE_CASE_   :      Optional[str]         =  None
				SCREAMING_SNAKE_CASE_   :      Optional[str]         =  None
				SCREAMING_SNAKE_CASE_   :      bool         =  True  # deprecated
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  None  # deprecated
				SCREAMING_SNAKE_CASE_   :      int         =  10 << 20  # 10MB
				SCREAMING_SNAKE_CASE_   :      Optional[bool]         =  None





class     snake_case	(				datasets.ArrowBasedBuilder  ):
				SCREAMING_SNAKE_CASE_   :      Tuple         =  JsonConfig

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						if self.config.block_size is not None:
								logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead")
								__lowerCAmelCase:       List[str]          =			self.config.block_size
						if self.config.use_threads is not True:
								logger.warning(
								    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore.")
						if self.config.newlines_in_values is not None:
								raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported")
						return datasets.DatasetInfo(features=self.config.features)

				def 							lowercase_     (	self							:				str     ,  UpperCamelCase__							:				Optional[Any])->   Any:



						'''simple docstring'''


						if not self.config.data_files:
								raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
						__lowerCAmelCase:       Dict          =			dl_manager.download_and_extract(self.config.data_files)
						if isinstance(UpperCamelCase__     ,  (str, list, tuple)):
								__lowerCAmelCase:       Optional[int]          =			data_files
								if isinstance(UpperCamelCase__     ,  UpperCamelCase__):
										__lowerCAmelCase:       Union[str, Any]          =			[files]
								__lowerCAmelCase:       Any          =			[dl_manager.iter_files(UpperCamelCase__) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,  gen_kwargs={"files": files})]
						__lowerCAmelCase:       Optional[Any]          =			[]
						for split_name, files in data_files.items():
								if isinstance(UpperCamelCase__     ,  UpperCamelCase__):
										__lowerCAmelCase:       Optional[int]          =			[files]
								__lowerCAmelCase:       int          =			[dl_manager.iter_files(UpperCamelCase__) for file in files]
								splits.append(datasets.SplitGenerator(name=UpperCamelCase__     ,  gen_kwargs={"files": files}))
						return splits

				def 							lowercase_     (	self							:				Tuple     ,  UpperCamelCase__							:				pa.Table)->   pa.Table:



						'''simple docstring'''


						if self.config.features is not None:
								# adding missing columns
								for column_name in set(self.config.features) - set(pa_table.column_names):
										__lowerCAmelCase:       str          =			self.config.features.arrow_schema.field(UpperCamelCase__).type
										__lowerCAmelCase:       Any          =			pa_table.append_column(UpperCamelCase__     ,  pa.array([None] * len(UpperCamelCase__)     ,  type=UpperCamelCase__))
								# more expensive cast to support nested structures with keys in a different order
								# allows str <-> int/float or str to Audio for example
								__lowerCAmelCase:       Dict          =			table_cast(UpperCamelCase__     ,  self.config.features.arrow_schema)
						return pa_table




				def 							lowercase_     (	self							:				Any     ,  UpperCamelCase__							:				Tuple)->   int:



						'''simple docstring'''


						for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__)):
								# If the file is one json object and if we need to look at the list of items in one specific field
								if self.config.field is not None:
										with open(UpperCamelCase__     ,  encoding=self.config.encoding     ,  errors=self.config.encoding_errors) as f:
												__lowerCAmelCase:       Optional[int]          =			json.load(UpperCamelCase__)

										# We keep only the field we are interested in
										__lowerCAmelCase:       str          =			dataset[self.config.field]

										# We accept two format: a list of dicts or a dict of lists
										if isinstance(UpperCamelCase__     ,  (list, tuple)):
												__lowerCAmelCase:       List[str]          =			set().union(*[row.keys() for row in dataset])
												__lowerCAmelCase:       Optional[Any]          =			{col: [row.get(UpperCamelCase__) for row in dataset] for col in keys}
										else:
												__lowerCAmelCase:       int          =			dataset
										__lowerCAmelCase:       Tuple          =			pa.Table.from_pydict(UpperCamelCase__)
										yield file_idx, self._cast_table(UpperCamelCase__)

								# If the file has one json object per line
								else:
										with open(UpperCamelCase__     ,  "rb") as f:
												__lowerCAmelCase:       Union[str, Any]          =			0
												# Use block_size equal to the chunk size divided by 32 to leverage multithreading
												# Set a default minimum value of 16kB if the chunk size is really small
												__lowerCAmelCase:       int          =			max(self.config.chunksize // 3_2     ,  1_6 << 1_0)
												__lowerCAmelCase:       List[Any]          =			(
												    self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
												)
												while True:
														__lowerCAmelCase:       Union[str, Any]          =			f.read(self.config.chunksize)
														if not batch:
																break
														# Finish current line
														try:
																batch += f.readline()
														except (AttributeError, io.UnsupportedOperation):
																batch += readline(UpperCamelCase__)
														# PyArrow only accepts utf-8 encoded bytes
														if self.config.encoding != "utf-8":
																__lowerCAmelCase:       List[Any]          =			batch.decode(self.config.encoding     ,  errors=UpperCamelCase__).encode("utf-8")
														try:
																while True:
																		try:
																				__lowerCAmelCase:       Optional[Any]          =			paj.read_json(
																				    io.BytesIO(UpperCamelCase__)     ,  read_options=paj.ReadOptions(block_size=UpperCamelCase__))
																				break
																		except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																				if (
																				    isinstance(UpperCamelCase__     ,  pa.ArrowInvalid)
																				    and "straddling" not in str(UpperCamelCase__)
																				    or block_size > len(UpperCamelCase__)
																				):
																						raise
																				else:
																						# Increase the block size in case it was too small.
																						# The block size will be reset for the next file.
																						logger.debug(
																						    f"Batch of {len(UpperCamelCase__)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.")
																						block_size *= 2
														except pa.ArrowInvalid as e:
																try:
																		with open(
																		    UpperCamelCase__     ,  encoding=self.config.encoding     ,  errors=self.config.encoding_errors) as f:
																				__lowerCAmelCase:       Union[str, Any]          =			json.load(UpperCamelCase__)
																except json.JSONDecodeError:
																		logger.error(f"Failed to read file '{file}' with error {type(UpperCamelCase__)}: {e}")
																		raise e
																# If possible, parse the file as a list of json objects and exit the loop
																if isinstance(UpperCamelCase__     ,  UpperCamelCase__):  # list is the only sequence type supported in JSON
																		try:
																				__lowerCAmelCase:       List[Any]          =			set().union(*[row.keys() for row in dataset])
																				__lowerCAmelCase:       Optional[int]          =			{col: [row.get(UpperCamelCase__) for row in dataset] for col in keys}
																				__lowerCAmelCase:       Any          =			pa.Table.from_pydict(UpperCamelCase__)
																		except (pa.ArrowInvalid, AttributeError) as e:
																				logger.error(f"Failed to read file '{file}' with error {type(UpperCamelCase__)}: {e}")
																				raise ValueError(f"Not able to read records in the JSON file at {file}.") from None
																		yield file_idx, self._cast_table(UpperCamelCase__)
																		break
																else:
																		logger.error(f"Failed to read file '{file}' with error {type(UpperCamelCase__)}: {e}")
																		raise ValueError(
																		    f"Not able to read records in the JSON file at {file}. "
																		    f"You should probably indicate the field of the JSON file containing your records. "
																		    f"This JSON file contain the following fields: {str(list(dataset.keys()))}. "
																		    f"Select the correct one and provide it as `field='XXX'` to the dataset loading method. ") from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
														yield (file_idx, batch_idx), self._cast_table(UpperCamelCase__)
														batch_idx += 1





"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  RoCBertTokenizer
				SCREAMING_SNAKE_CASE_   :      Any         =  None
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  True
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  filter_non_english

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						super().setUp()

						__lowerCAmelCase:       Optional[Any]          =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
						__lowerCAmelCase:       List[Any]          =			{}
						__lowerCAmelCase:       Dict          =			{}
						for i, value in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       List[Any]          =			i
								__lowerCAmelCase:       Union[str, Any]          =			i
						__lowerCAmelCase:       List[str]          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"])
						with open(self.vocab_file     ,  "w"     ,  encoding="utf-8") as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
						with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8") as word_shape_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)
						with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8") as word_pronunciation_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.tokenize("你好[SEP]你是谁")
						self.assertListEqual(UpperCamelCase__     ,  ["你", "好", "[SEP]", "你", "是", "谁"])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")     ,  ["ah", "\u535A", "\u63A8", "zz"])

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["hello", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hällo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["h\u00E9llo"])

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				str)->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[Any])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Tuple)->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  never_split=["[UNK]"])

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

				def 							lowercase_     (	self							:				List[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__lowerCAmelCase:       int          =			{}
						for i, token in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       Optional[Any]          =			i
						__lowerCAmelCase:       str          =			RoCBertWordpieceTokenizer(vocab=UpperCamelCase__     ,  unk_token="[UNK]")

						self.assertListEqual(tokenizer.tokenize("")     ,  [])

						self.assertListEqual(tokenizer.tokenize("unwanted running")     ,  ["un", "##want", "##ed", "runn", "##ing"])

						self.assertListEqual(tokenizer.tokenize("unwantedX running")     ,  ["[UNK]", "runn", "##ing"])

				def 							lowercase_     (	self							:				Optional[Any])->   Dict:



						'''simple docstring'''


						self.assertTrue(_is_whitespace(" "))
						self.assertTrue(_is_whitespace("\t"))
						self.assertTrue(_is_whitespace("\r"))
						self.assertTrue(_is_whitespace("\n"))
						self.assertTrue(_is_whitespace("\u00A0"))

						self.assertFalse(_is_whitespace("A"))
						self.assertFalse(_is_whitespace("-"))

				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						self.assertTrue(_is_control("\u0005"))

						self.assertFalse(_is_control("A"))
						self.assertFalse(_is_control(" "))
						self.assertFalse(_is_control("\t"))
						self.assertFalse(_is_control("\r"))

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						self.assertTrue(_is_punctuation("-"))
						self.assertTrue(_is_punctuation("$"))
						self.assertTrue(_is_punctuation("`"))
						self.assertTrue(_is_punctuation("."))

						self.assertFalse(_is_punctuation("A"))
						self.assertFalse(_is_punctuation(" "))

				def 							lowercase_     (	self							:				Dict)->   int:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

						if self.test_rust_tokenizer:
								__lowerCAmelCase:       Any          =			self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       str          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowerCAmelCase:       Tuple          =			tokenizer_r.encode_plus(
										    UpperCamelCase__     ,  return_attention_mask=UpperCamelCase__     ,  return_token_type_ids=UpperCamelCase__     ,  return_offsets_mapping=UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  )

										__lowerCAmelCase:       str          =			tokenizer_r.do_lower_case if hasattr(UpperCamelCase__     ,  "do_lower_case") else False
										__lowerCAmelCase:       List[Any]          =			(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "Allen"),
										        ((2_1, 2_3), "##NL"),
										        ((2_3, 2_4), "##P"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "allen"),
										        ((2_1, 2_3), "##nl"),
										        ((2_3, 2_4), "##p"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"])

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			["的", "人", "有"]
						__lowerCAmelCase:       int          =			"".join(UpperCamelCase__)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Tuple          =			True
										__lowerCAmelCase:       str          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Union[str, Any]          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       List[Any]          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Any          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       List[str]          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

										__lowerCAmelCase:       int          =			False
										__lowerCAmelCase:       Any          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       str          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase:       Dict          =			[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCamelCase__)
										]
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				@slow
				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Dict          =			tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase__)
						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase__)

						__lowerCAmelCase:       Tuple          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]




				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.get_tokenizers(do_lower_case=UpperCamelCase__)
						for tokenizer in tokenizers:
								with self.subTest(f"{tokenizer.__class__.__name__}"):
										__lowerCAmelCase:       str          =			"你好，你是谁"
										__lowerCAmelCase:       Dict          =			tokenizer.tokenize(UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			tokenizer.convert_tokens_to_ids(UpperCamelCase__)
										__lowerCAmelCase:       Optional[Any]          =			tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			tokenizer.prepare_for_model(
										    UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			tokenizer.encode_plus(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)


