




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase					:       Dict    					=      get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
@require_tokenizers
class 		__lowerCAmelCase	(	__magic_name__      ,       unittest.TestCase ):
							UpperCamelCase__					     =  XLNetTokenizer
							UpperCamelCase__					     =  XLNetTokenizerFast
							UpperCamelCase__					     =  True
							UpperCamelCase__					     =  True
							def       lowerCamelCase__							(						self						:Union[str, Any]				):







								'''simple docstring'''







								super().setUp()

								# We have a SentencePiece fixture for testing
								a										=							XLNetTokenizer(__magic_name__		,      keep_accents=__magic_name__				)
								tokenizer.sanitize_special_tokens()
								tokenizer.save_pretrained(self.tmpdirname				)
							def       lowerCamelCase__							(						self						:Optional[Any]				):







								'''simple docstring'''







								a										=							"""<s>"""
								a										=							1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__				)		,      __magic_name__				)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__				)		,      __magic_name__				)
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								a										=							list(self.get_tokenizer().get_vocab().keys()				)

								self.assertEqual(vocab_keys[0]		,      """<unk>"""				)
								self.assertEqual(vocab_keys[1]		,      """<s>"""				)
								self.assertEqual(vocab_keys[-1]		,      """<eod>"""				)
								self.assertEqual(len(__magic_name__				)		,      1006				)
							def       lowerCamelCase__							(						self						:Optional[Any]				):







								'''simple docstring'''







								self.assertEqual(self.get_tokenizer().vocab_size		,      1000				)
							def       lowerCamelCase__							(						self						:str				):







								'''simple docstring'''







								a										=							XLNetTokenizer(__magic_name__		,      keep_accents=__magic_name__				)

								a										=							tokenizer.tokenize("""This is a test"""				)
								self.assertListEqual(__magic_name__		,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)		,      [285, 46, 10, 170, 382]				)

								a										=							tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    __magic_name__		,      [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """é""",
								        """.""",
								    ]		,      )
								a										=							tokenizer.convert_tokens_to_ids(__magic_name__				)
								self.assertListEqual(__magic_name__		,      [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]				)

								a										=							tokenizer.convert_ids_to_tokens(__magic_name__				)
								self.assertListEqual(
								    __magic_name__		,      [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """<unk>""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """<unk>""",
								        """.""",
								    ]		,      )
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								a										=							XLNetTokenizer(__magic_name__		,      do_lower_case=__magic_name__				)
								a										=							tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    __magic_name__		,      [
								        SPIECE_UNDERLINE + """""",
								        """i""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """se""",
								        """.""",
								    ]		,      )
								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				)		,      ["""▁he""", """ll""", """o"""]				)
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								a										=							XLNetTokenizer(__magic_name__		,      do_lower_case=__magic_name__				)
								a										=							tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    __magic_name__		,      [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """se""",
								        """.""",
								    ]		,      )
							@slow
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								a										=							XLNetTokenizer.from_pretrained("""xlnet-base-cased"""				)

								a										=							tokenizer.encode("""sequence builders"""		,      add_special_tokens=__magic_name__				)
								a										=							tokenizer.encode("""multi-sequence build"""		,      add_special_tokens=__magic_name__				)

								a										=							tokenizer.build_inputs_with_special_tokens(__magic_name__				)
								a										=							tokenizer.build_inputs_with_special_tokens(__magic_name__		,      __magic_name__				)

								assert encoded_sentence == text + [4, 3]
								assert encoded_pair == text + [4] + text_a + [4, 3]





							@slow
							def       lowerCamelCase__							(						self						:Optional[int]				):







								'''simple docstring'''







								a										=							{"""input_ids""": [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=__magic_name__		,      model_name="""xlnet-base-cased"""		,      revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""		,      )







from __future__ import annotations

from collections.abc import Iterator

class 		__lowerCAmelCase	:
							def __init__(						self						:Optional[Any]		,      __magic_name__						:int				):







								'''simple docstring'''







								a										=							value
								a										=							None
								a										=							None







class 		__lowerCAmelCase	:
							def __init__(						self						:str		,      __magic_name__						:Node				):







								'''simple docstring'''







								a										=							tree
							def       lowerCamelCase__							(						self						:str		,      __magic_name__						:Node | None				):







								'''simple docstring'''







								if node is None:
									return 0
								return node.value + (
								    self.depth_first_search(node.left				) + self.depth_first_search(node.right				)
								)





							def __iter__(						self						:Tuple				):







								'''simple docstring'''







								yield self.depth_first_search(self.tree				)


if __name__ == "__main__":
		import doctest

		doctest.testmod()

