'''simple docstring'''




import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase						: List[str]						=						get_tests_dir("fixtures/test_sentencepiece.model")





@require_sentencepiece
@require_tokenizers
class     __magic_name__						(    _UpperCAmelCase,     unittest.TestCase):
			UpperCamelCase__				=       XLMRobertaTokenizer
			UpperCamelCase__				=       XLMRobertaTokenizerFast
			UpperCamelCase__				=       True
			UpperCamelCase__				=       True







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Optional[Any]    ):
										super().setUp()

										# We have a SentencePiece fixture for testing
										lowercase_					:			Optional[Any]										=       XLMRobertaTokenizer(lowercase_				,			keep_accents=lowercase_    )
										tokenizer.save_pretrained(self.tmpdirname    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Optional[Any]    ):
										lowercase_					:			Union[str, Any]										=       """<pad>"""
										lowercase_					:			List[str]										=       1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_    )				,			lowercase_    )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_    )				,			lowercase_    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Union[str, Any]    ):
										lowercase_					:			Tuple										=       list(self.get_tokenizer().get_vocab().keys()    )

										self.assertEqual(vocab_keys[0]				,			"""<s>"""    )
										self.assertEqual(vocab_keys[1]				,			"""<pad>"""    )
										self.assertEqual(vocab_keys[-1]				,			"""<mask>"""    )
										self.assertEqual(len(lowercase_    )				,			1002    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Tuple    ):
										self.assertEqual(self.get_tokenizer().vocab_size				,			1002    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Union[str, Any]    ):
										lowercase_					:			int										=       XLMRobertaTokenizer(lowercase_				,			keep_accents=lowercase_    )

										lowercase_					:			str										=       tokenizer.tokenize("""This is a test"""    )
										self.assertListEqual(lowercase_				,			["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowercase_    )				,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				,			)

										lowercase_					:			Union[str, Any]										=       tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
										self.assertListEqual(
										    lowercase_				,			[
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """9""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """é""",
										        """.""",
										    ]				,			)
										lowercase_					:			Union[str, Any]										=       tokenizer.convert_tokens_to_ids(lowercase_    )
										self.assertListEqual(
										    lowercase_				,			[
										        value + tokenizer.fairseq_offset
										        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
										        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
										    ]				,			)

										lowercase_					:			List[str]										=       tokenizer.convert_ids_to_tokens(lowercase_    )
										self.assertListEqual(
										    lowercase_				,			[
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """<unk>""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """<unk>""",
										        """.""",
										    ]				,			)







			def 	SCREAMING_SNAKE_CASE_     (	self				:  List[str]    ):
										if not self.test_slow_tokenizer:
																	# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																	return

										lowercase_					:			int										=       (self.rust_tokenizer_class, """hf-internal-testing/tiny-xlm-roberta""", {})
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																								lowercase_					:			Dict										=       self.rust_tokenizer_class.from_pretrained(lowercase_				,			**lowercase_    )
																								lowercase_					:			Optional[Any]										=       self.tokenizer_class.from_pretrained(lowercase_				,			**lowercase_    )

																								lowercase_					:			Optional[int]										=       tempfile.mkdtemp()

																								lowercase_					:			List[str]										=       tokenizer_r.save_pretrained(lowercase_    )
																								lowercase_					:			Dict										=       tokenizer_p.save_pretrained(lowercase_    )

																								# Checks it save with the same files + the tokenizer.json file for the fast one
																								self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )
																								lowercase_					:			List[Any]										=       tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f    )
																								self.assertSequenceEqual(lowercase_				,			lowercase_    )

																								# Checks everything loads correctly in the same way
																								lowercase_					:			Tuple										=       tokenizer_r.from_pretrained(lowercase_    )
																								lowercase_					:			Tuple										=       tokenizer_p.from_pretrained(lowercase_    )

																								# Check special tokens are set accordingly on Rust and Python
																								for key in tokenizer_pp.special_tokens_map:
																															self.assertTrue(hasattr(lowercase_				,			lowercase_    )    )
																															# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																															# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																								shutil.rmtree(lowercase_    )

																								# Save tokenizer rust, legacy_format=True
																								lowercase_					:			Optional[Any]										=       tempfile.mkdtemp()

																								lowercase_					:			Optional[int]										=       tokenizer_r.save_pretrained(lowercase_				,			legacy_format=lowercase_    )
																								lowercase_					:			str										=       tokenizer_p.save_pretrained(lowercase_    )

																								# Checks it save with the same files
																								self.assertSequenceEqual(lowercase_				,			lowercase_    )

																								# Checks everything loads correctly in the same way
																								lowercase_					:			Union[str, Any]										=       tokenizer_r.from_pretrained(lowercase_    )
																								lowercase_					:			List[str]										=       tokenizer_p.from_pretrained(lowercase_    )

																								# Check special tokens are set accordingly on Rust and Python
																								for key in tokenizer_pp.special_tokens_map:
																															self.assertTrue(hasattr(lowercase_				,			lowercase_    )    )

																								shutil.rmtree(lowercase_    )

																								# Save tokenizer rust, legacy_format=False
																								lowercase_					:			Dict										=       tempfile.mkdtemp()

																								lowercase_					:			Optional[Any]										=       tokenizer_r.save_pretrained(lowercase_				,			legacy_format=lowercase_    )
																								lowercase_					:			Union[str, Any]										=       tokenizer_p.save_pretrained(lowercase_    )

																								# Checks it saved the tokenizer.json file
																								self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )

																								# Checks everything loads correctly in the same way
																								lowercase_					:			int										=       tokenizer_r.from_pretrained(lowercase_    )
																								lowercase_					:			List[Any]										=       tokenizer_p.from_pretrained(lowercase_    )

																								# Check special tokens are set accordingly on Rust and Python
																								for key in tokenizer_pp.special_tokens_map:
																															self.assertTrue(hasattr(lowercase_				,			lowercase_    )    )

																								shutil.rmtree(lowercase_    )







			@cached_property
			def 	SCREAMING_SNAKE_CASE_     (	self				:  Tuple    ):
										return XLMRobertaTokenizer.from_pretrained("""xlm-roberta-base"""    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  str    ):
										with tempfile.NamedTemporaryFile() as f:
																	shutil.copyfile(lowercase_				,			f.name    )
																	lowercase_					:			int										=       XLMRobertaTokenizer(f.name				,			keep_accents=lowercase_    )
																	lowercase_					:			Any										=       pickle.dumps(lowercase_    )
										pickle.loads(lowercase_    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Optional[int]    ):
										if not self.test_rust_tokenizer:
																	return

										lowercase_					:			Any										=       self.get_tokenizer()
										lowercase_					:			Tuple										=       self.get_rust_tokenizer()

										lowercase_					:			List[Any]										=       """I was born in 92000, and this is falsé."""

										lowercase_					:			Tuple										=       tokenizer.tokenize(lowercase_    )
										lowercase_					:			List[str]										=       rust_tokenizer.tokenize(lowercase_    )
										self.assertListEqual(lowercase_				,			lowercase_    )

										lowercase_					:			Optional[Any]										=       tokenizer.encode(lowercase_				,			add_special_tokens=lowercase_    )
										lowercase_					:			Union[str, Any]										=       rust_tokenizer.encode(lowercase_				,			add_special_tokens=lowercase_    )
										self.assertListEqual(lowercase_				,			lowercase_    )

										lowercase_					:			Tuple										=       self.get_rust_tokenizer()
										lowercase_					:			Any										=       tokenizer.encode(lowercase_    )
										lowercase_					:			Dict										=       rust_tokenizer.encode(lowercase_    )
										self.assertListEqual(lowercase_				,			lowercase_    )







			@slow
			def 	SCREAMING_SNAKE_CASE_     (	self				:  Dict    ):
										lowercase_					:			Any										=       """Hello World!"""
										lowercase_					:			int										=       [0, 35378, 6661, 38, 2]
										# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
										# xlmr.eval()
										# xlmr.encode(symbols)

										self.assertListEqual(lowercase_				,			self.big_tokenizer.encode(lowercase_    )    )







			@slow
			def 	SCREAMING_SNAKE_CASE_     (	self				:  Tuple    ):
										lowercase_					:			str										=       (
										    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
										    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
										)
										lowercase_					:			Union[str, Any]										=       [
										    0,
										    3293,
										    83,
										    10,
										    4552,
										    4989,
										    7986,
										    678,
										    10,
										    5915,
										    111,
										    179459,
										    124850,
										    4,
										    6044,
										    237,
										    12,
										    6,
										    5,
										    6,
										    4,
										    6780,
										    705,
										    15,
										    1388,
										    44,
										    378,
										    10114,
										    711,
										    152,
										    20,
										    6,
										    5,
										    22376,
										    642,
										    1221,
										    15190,
										    34153,
										    450,
										    5608,
										    959,
										    1119,
										    57702,
										    136,
										    186,
										    47,
										    1098,
										    29367,
										    47,
										    # 4426, # What fairseq tokenizes from "<unk>": "_<"
										    # 3678, # What fairseq tokenizes from "<unk>": "unk"
										    # 2740, # What fairseq tokenizes from "<unk>": ">"
										    3,  # What we tokenize from "<unk>": "<unk>"
										    6,  # Residue from the tokenization: an extra sentencepiece underline
										    4,
										    6044,
										    237,
										    6284,
										    50901,
										    528,
										    31,
										    90,
										    34,
										    927,
										    2,
										]
										# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
										# xlmr.eval()
										# xlmr.encode(symbols)

										self.assertListEqual(lowercase_				,			self.big_tokenizer.encode(lowercase_    )    )







			@slow
			def 	SCREAMING_SNAKE_CASE_     (	self				:  List[str]    ):
										# fmt: off
										lowercase_					:			List[Any]										=       {"""input_ids""": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowercase_				,			model_name="""xlm-roberta-base"""				,			revision="""d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"""				,			)






'''simple docstring'''




_lowercase						: str						=						tuple[float, float, float]
_lowercase						: List[Any]						=						tuple[float, float, float]




def 							lowerCamelCase	(      UpperCAmelCase__       :					Pointad      ,      UpperCAmelCase__       :					Pointad )			->      Vectorad:
							lowercase_					:			List[str]										=       end_pointa[0] - end_pointa[0]
							lowercase_					:			Union[str, Any]										=       end_pointa[1] - end_pointa[1]
							lowercase_					:			List[Any]										=       end_pointa[2] - end_pointa[2]
							return (x, y, z)




def 							lowerCamelCase	(      UpperCAmelCase__       :					Vectorad      ,      UpperCAmelCase__       :					Vectorad )			->      Vectorad:
							lowercase_					:			List[Any]										=       ab[1] * ac[2] - ab[2] * ac[1]  # *i
							lowercase_					:			Union[str, Any]										=       (ab[0] * ac[2] - ab[2] * ac[0]) * -1  # *j
							lowercase_					:			List[str]										=       ab[0] * ac[1] - ab[1] * ac[0]  # *k
							return (x, y, z)




def 							lowerCamelCase	(      UpperCAmelCase__       :					Vectorad      ,      UpperCAmelCase__       :					int )			->      bool:
							return tuple(round(UpperCAmelCase__      ,      UpperCAmelCase__ ) for x in vector ) == (0, 0, 0)




def 							lowerCamelCase	(      UpperCAmelCase__       :					Pointad      ,      UpperCAmelCase__       :					Pointad      ,      UpperCAmelCase__       :					Pointad      ,      UpperCAmelCase__       :					int = 10 )			->      bool:
							lowercase_					:			Dict										=       create_vector(UpperCAmelCase__      ,      UpperCAmelCase__ )
							lowercase_					:			Optional[int]										=       create_vector(UpperCAmelCase__      ,      UpperCAmelCase__ )
							return is_zero_vector(get_ad_vectors_cross(UpperCAmelCase__      ,      UpperCAmelCase__ )      ,      UpperCAmelCase__ )





