


import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase					:  Optional[Any]			=						get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class     __lowercase    (				_snake_case       ,				unittest.TestCase   ):



		"""simple docstring"""

		UpperCamelCase			: str           =						ReformerTokenizer
		UpperCamelCase			: Optional[int]           =						ReformerTokenizerFast
		UpperCamelCase			: str           =						True
		UpperCamelCase			: Tuple           =						False
		UpperCamelCase			: str           =						True







		def  __A					(      self     )   ->      int:







					'''simple docstring'''







					super().setUp()

					lowerCamelCase  						=   ReformerTokenizer(__snake_case						,	keep_accents=__snake_case     )
					tokenizer.save_pretrained(self.tmpdirname     )







		def  __A					(      self     )   ->      Tuple:







					'''simple docstring'''







					lowerCamelCase  						=   """<s>"""
					lowerCamelCase  						=   1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case     )						,	__snake_case     )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case     )						,	__snake_case     )







		def  __A					(      self     )   ->      List[str]:







					'''simple docstring'''







					lowerCamelCase  						=   list(self.get_tokenizer().get_vocab().keys()     )

					self.assertEqual(vocab_keys[0]						,	"""<unk>"""     )
					self.assertEqual(vocab_keys[1]						,	"""<s>"""     )
					self.assertEqual(vocab_keys[-1]						,	"""j"""     )
					self.assertEqual(len(__snake_case     )						,	10_00     )







		def  __A					(      self     )   ->      Any:







					'''simple docstring'''







					self.assertEqual(self.get_tokenizer().vocab_size						,	10_00     )







		def  __A					(      self     )   ->      Optional[int]:







					'''simple docstring'''







					if not self.test_rust_tokenizer:
								return

					lowerCamelCase  						=   self.get_tokenizer()
					lowerCamelCase  						=   self.get_rust_tokenizer()

					lowerCamelCase  						=   """I was born in 92000, and this is falsé."""

					lowerCamelCase  						=   tokenizer.tokenize(__snake_case     )
					lowerCamelCase  						=   rust_tokenizer.tokenize(__snake_case     )
					self.assertListEqual(__snake_case						,	__snake_case     )

					lowerCamelCase  						=   tokenizer.encode(__snake_case						,	add_special_tokens=__snake_case     )
					lowerCamelCase  						=   rust_tokenizer.encode(__snake_case						,	add_special_tokens=__snake_case     )
					self.assertListEqual(__snake_case						,	__snake_case     )

					lowerCamelCase  						=   self.get_rust_tokenizer()
					lowerCamelCase  						=   tokenizer.encode(__snake_case     )
					lowerCamelCase  						=   rust_tokenizer.encode(__snake_case     )
					self.assertListEqual(__snake_case						,	__snake_case     )







		def  __A					(      self						,	A=15     )   ->      List[str]:







					'''simple docstring'''







					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
											lowerCamelCase  						=   self.rust_tokenizer_class.from_pretrained(__snake_case						,	**__snake_case     )

											# Simple input
											lowerCamelCase  						=   """This is a simple input"""
											lowerCamelCase  						=   ["""This is a simple input 1""", """This is a simple input 2"""]
											lowerCamelCase  						=   ("""This is a simple input""", """This is a pair""")
											lowerCamelCase  						=   [
											    ("""This is a simple input 1""", """This is a simple input 2"""),
											    ("""This is a simple pair 1""", """This is a simple pair 2"""),
											]

											# Simple input tests
											self.assertRaises(__snake_case						,	tokenizer_r.encode						,	__snake_case						,	max_length=__snake_case						,	padding="""max_length"""     )

											# Simple input
											self.assertRaises(__snake_case						,	tokenizer_r.encode_plus						,	__snake_case						,	max_length=__snake_case						,	padding="""max_length"""     )

											# Simple input
											self.assertRaises(
											    __snake_case						,	tokenizer_r.batch_encode_plus						,	__snake_case						,	max_length=__snake_case						,	padding="""max_length"""						,	)

											# Pair input
											self.assertRaises(__snake_case						,	tokenizer_r.encode						,	__snake_case						,	max_length=__snake_case						,	padding="""max_length"""     )

											# Pair input
											self.assertRaises(__snake_case						,	tokenizer_r.encode_plus						,	__snake_case						,	max_length=__snake_case						,	padding="""max_length"""     )

											# Pair input
											self.assertRaises(
											    __snake_case						,	tokenizer_r.batch_encode_plus						,	__snake_case						,	max_length=__snake_case						,	padding="""max_length"""						,	)







		def  __A					(      self     )   ->      Any:







					'''simple docstring'''







					pass







		def  __A					(      self     )   ->      str:







					'''simple docstring'''







					lowerCamelCase  						=   ReformerTokenizer(__snake_case						,	keep_accents=__snake_case     )

					lowerCamelCase  						=   tokenizer.tokenize("""This is a test"""     )
					self.assertListEqual(__snake_case						,	["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]     )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__snake_case     )						,	[2_85, 46, 10, 1_70, 3_82]						,	)

					lowerCamelCase  						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""     )
					self.assertListEqual(
					    __snake_case						,	[
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]						,	)
					lowerCamelCase  						=   tokenizer.convert_tokens_to_ids(__snake_case     )
					self.assertListEqual(
					    __snake_case						,	[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]						,	)

					lowerCamelCase  						=   tokenizer.convert_ids_to_tokens(__snake_case     )
					self.assertListEqual(
					    __snake_case						,	[
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]						,	)







		@cached_property
		def  __A					(      self     )   ->      Union[str, Any]:







					'''simple docstring'''







					return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""     )







		@slow
		def  __A					(      self     )   ->      List[Any]:







					'''simple docstring'''







					lowerCamelCase  						=   """Hello World!"""
					lowerCamelCase  						=   [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

					self.assertListEqual(__snake_case						,	self.big_tokenizer.encode(__snake_case     )     )







		@slow
		def  __A					(      self     )   ->      List[str]:







					'''simple docstring'''







					lowerCamelCase  						=   (
					    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
					    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
					)
					lowerCamelCase  						=   [
					    1_08,
					    2_65,
					    24,
					    1_11,
					    4,
					    2_58,
					    1_56,
					    35,
					    28,
					    2_75,
					    3,
					    2_59,
					    2_97,
					    2_60,
					    84,
					    4,
					    35,
					    1_10,
					    44,
					    8,
					    2_59,
					    91,
					    2_68,
					    21,
					    11,
					    2_09,
					    2_74,
					    1_09,
					    2_66,
					    2_77,
					    1_17,
					    86,
					    93,
					    3_15,
					    2_58,
					    2_78,
					    2_58,
					    2_77,
					    2_58,
					    0,
					    2_58,
					    2_88,
					    2_58,
					    3_19,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    2_87,
					    2_58,
					    3_15,
					    2_58,
					    2_89,
					    2_58,
					    2_78,
					    99,
					    2_69,
					    2_66,
					    2_62,
					    8,
					    2_59,
					    2_41,
					    4,
					    2_17,
					    2_30,
					    2_68,
					    2_66,
					    55,
					    1_68,
					    1_06,
					    75,
					    1_93,
					    2_66,
					    2_23,
					    27,
					    49,
					    26,
					    2_82,
					    25,
					    2_64,
					    2_99,
					    19,
					    26,
					    0,
					    2_58,
					    2_77,
					    1_17,
					    86,
					    93,
					    1_76,
					    1_83,
					    2_70,
					    11,
					    2_62,
					    42,
					    61,
					    2_65,
					]

					self.assertListEqual(__snake_case						,	self.big_tokenizer.encode(__snake_case     )     )







		@require_torch
		@slow
		def  __A					(      self     )   ->      Any:







					'''simple docstring'''







					import torch

					from transformers import ReformerConfig, ReformerModel

					# Build sequence
					lowerCamelCase  						=   list(self.big_tokenizer.get_vocab().keys()     )[:10]
					lowerCamelCase  						=   """ """.join(__snake_case     )
					lowerCamelCase  						=   self.big_tokenizer.encode_plus(__snake_case						,	return_tensors="""pt"""     )
					lowerCamelCase  						=   self.big_tokenizer.batch_encode_plus([sequence, sequence]						,	return_tensors="""pt"""     )

					lowerCamelCase  						=   ReformerConfig()
					# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
					lowerCamelCase  						=   encoded_sequence["""input_ids"""].shape
					lowerCamelCase  						=   ReformerModel(__snake_case     )

					# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
					assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

					with torch.no_grad():
								model(**__snake_case     )
								model(**__snake_case     )







		@slow
		def  __A					(      self     )   ->      List[str]:







					'''simple docstring'''







					lowerCamelCase  						=   {"""input_ids""": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					# This tokenizer does not know some characters like ")".
					# That is the reason why we use very simple texts here.
					# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
					lowerCamelCase  						=   [
					    """This is a very simple sentence.""",
					    """The quick brown fox jumps over the lazy dog.""",
					]

					self.tokenizer_integration_test_util(
					    expected_encoding=__snake_case						,	model_name="""google/reformer-crime-and-punishment"""						,	revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""						,	padding=__snake_case						,	sequences=__snake_case						,	)





'''simple docstring'''


import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
				import ctypes
				import msvcrt  # noqa





				class     lowercase__  (						ctypes.Structure					):





											'''simple docstring'''


											A_							:      Optional[Any]      	=  [("""size""", ctypes.c_int), ("""visible""", ctypes.c_byte)]







def 			snake_case_   (  ):







				"""simple docstring"""





				if os.name == "nt":
								_SCREAMING_SNAKE_CASE :       Tuple     		=      CursorInfo()
								_SCREAMING_SNAKE_CASE :       Tuple     		=      ctypes.windll.kernelaa.GetStdHandle(-11     )
								ctypes.windll.kernelaa.GetConsoleCursorInfo(SCREAMING_SNAKE_CASE__							,      ctypes.byref(SCREAMING_SNAKE_CASE__     )     )
								_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      False
								ctypes.windll.kernelaa.SetConsoleCursorInfo(SCREAMING_SNAKE_CASE__							,      ctypes.byref(SCREAMING_SNAKE_CASE__     )     )
				elif os.name == "posix":
								sys.stdout.write("""\033[?25l"""     )
								sys.stdout.flush()







def 			snake_case_   (  ):







				"""simple docstring"""





				if os.name == "nt":
								_SCREAMING_SNAKE_CASE :       int     		=      CursorInfo()
								_SCREAMING_SNAKE_CASE :       List[str]     		=      ctypes.windll.kernelaa.GetStdHandle(-11     )
								ctypes.windll.kernelaa.GetConsoleCursorInfo(SCREAMING_SNAKE_CASE__							,      ctypes.byref(SCREAMING_SNAKE_CASE__     )     )
								_SCREAMING_SNAKE_CASE :       Tuple     		=      True
								ctypes.windll.kernelaa.SetConsoleCursorInfo(SCREAMING_SNAKE_CASE__							,      ctypes.byref(SCREAMING_SNAKE_CASE__     )     )
				elif os.name == "posix":
								sys.stdout.write("""\033[?25h"""     )
								sys.stdout.flush()







@contextmanager
def 			snake_case_   (  ):







				"""simple docstring"""





				try:
								hide_cursor()
								yield
				finally:
								show_cursor()




