


"""simple docstring"""





import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__						:   str			 = '''▁'''

snake_case__						:   Optional[Any]			 = get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
@require_tokenizers
class 						snake_case_(				__snake_case							,    unittest.TestCase	):
     __UpperCamelCase     =    BigBirdTokenizer
     __UpperCamelCase     =    BigBirdTokenizerFast
     __UpperCamelCase     =    True
     __UpperCamelCase     =    True






     def    lowerCamelCase__   (		self     :    Union[str, Any] ):
           super().setUp()

           lowerCAmelCase    :		int				=      self.tokenizer_class(UpperCamelCase__    , keep_accents=UpperCamelCase__ )
           tokenizer.save_pretrained(self.tmpdirname )






     def    lowerCamelCase__   (		self     :    Union[str, Any] ):
           lowerCAmelCase    :		Dict				=      "<s>"
           lowerCAmelCase    :		Union[str, Any]				=      1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__ )    , UpperCamelCase__ )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__ )    , UpperCamelCase__ )






     def    lowerCamelCase__   (		self     :    List[Any] ):
           lowerCAmelCase    :		int				=      list(self.get_tokenizer().get_vocab().keys() )

           self.assertEqual(vocab_keys[0]    , '''<unk>''' )
           self.assertEqual(vocab_keys[1]    , '''<s>''' )
           self.assertEqual(vocab_keys[-1]    , '''[MASK]''' )
           self.assertEqual(len(UpperCamelCase__ )    , 1_0_0_4 )






     def    lowerCamelCase__   (		self     :    Tuple ):
           self.assertEqual(self.get_tokenizer().vocab_size    , 1_0_0_0 )






     def    lowerCamelCase__   (		self     :    List[str] ):


           if not self.test_rust_tokenizer:
                 return

           lowerCAmelCase    :		str				=      self.get_tokenizer()
           lowerCAmelCase    :		Any				=      self.get_rust_tokenizer()

           lowerCAmelCase    :		str				=      "I was born in 92000, and this is falsé."

           lowerCAmelCase    :		str				=      tokenizer.tokenize(UpperCamelCase__ )
           lowerCAmelCase    :		Any				=      rust_tokenizer.tokenize(UpperCamelCase__ )
           self.assertListEqual(UpperCamelCase__    , UpperCamelCase__ )

           lowerCAmelCase    :		Dict				=      tokenizer.encode(UpperCamelCase__    , add_special_tokens=UpperCamelCase__ )
           lowerCAmelCase    :		List[str]				=      rust_tokenizer.encode(UpperCamelCase__    , add_special_tokens=UpperCamelCase__ )
           self.assertListEqual(UpperCamelCase__    , UpperCamelCase__ )

           lowerCAmelCase    :		Union[str, Any]				=      self.get_rust_tokenizer()
           lowerCAmelCase    :		Any				=      tokenizer.encode(UpperCamelCase__ )
           lowerCAmelCase    :		Union[str, Any]				=      rust_tokenizer.encode(UpperCamelCase__ )
           self.assertListEqual(UpperCamelCase__    , UpperCamelCase__ )






     def    lowerCamelCase__   (		self     :    Dict ):
           lowerCAmelCase    :		str				=      BigBirdTokenizer(UpperCamelCase__    , keep_accents=UpperCamelCase__ )

           lowerCAmelCase    :		int				=      tokenizer.tokenize('''This is a test''' )
           self.assertListEqual(UpperCamelCase__    , ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(UpperCamelCase__ )    , [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]    , )

           lowerCAmelCase    :		int				=      tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
           self.assertListEqual(
               UpperCamelCase__    , [
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''9''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''é''',
                   '''.''',
               ]    , )
           lowerCAmelCase    :		str				=      tokenizer.convert_tokens_to_ids(UpperCamelCase__ )
           self.assertListEqual(
               UpperCamelCase__    , [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]    , )

           lowerCAmelCase    :		Optional[int]				=      tokenizer.convert_ids_to_tokens(UpperCamelCase__ )
           self.assertListEqual(
               UpperCamelCase__    , [
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''<unk>''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''<unk>''',
                   '''.''',
               ]    , )






     @cached_property
     def    lowerCamelCase__   (		self     :    Dict ):
           return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''' )






     @slow
     def    lowerCamelCase__   (		self     :    Any ):
           lowerCAmelCase    :		Optional[int]				=      "Hello World!"
           lowerCAmelCase    :		Union[str, Any]				=      [6_5, 1_8_5_3_6, 2_2_6_0, 1_0_1, 6_6]

           self.assertListEqual(UpperCamelCase__    , self.big_tokenizer.encode(UpperCamelCase__ ) )






     @slow
     def    lowerCamelCase__   (		self     :    List[Any] ):
           lowerCAmelCase    :		int				=      (
               "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
               " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
           )
           # fmt: off
           lowerCAmelCase    :		Any				=      [6_5, 8_7_1, 4_1_9, 3_5_8, 9_4_6, 9_9_1, 2_5_2_1, 4_5_2, 3_5_8, 1_3_5_7, 3_8_7, 7_7_5_1, 3_5_3_6, 1_1_2, 9_8_5, 4_5_6, 1_2_6, 8_6_5, 9_3_8, 5_4_0_0, 5_7_3_4, 4_5_8, 1_3_6_8, 4_6_7, 7_8_6, 2_4_6_2, 5_2_4_6, 1_1_5_9, 6_3_3, 8_6_5, 4_5_1_9, 4_5_7, 5_8_2, 8_5_2, 2_5_5_7, 4_2_7, 9_1_6, 5_0_8, 4_0_5, 3_4_3_2_4, 4_9_7, 3_9_1, 4_0_8, 1_1_3_4_2, 1_2_4_4, 3_8_5, 1_0_0, 9_3_8, 9_8_5, 4_5_6, 5_7_4, 3_6_2, 1_2_5_9_7, 3_2_0_0, 3_1_2_9, 1_1_7_2, 6_6]  # noqa: E231
           # fmt: on
           self.assertListEqual(UpperCamelCase__    , self.big_tokenizer.encode(UpperCamelCase__ ) )






     @require_torch
     @slow
     def    lowerCamelCase__   (		self     :    Union[str, Any] ):
           import torch

           from transformers import BigBirdConfig, BigBirdModel

           # Build sequence
           lowerCAmelCase    :		Any				=      list(self.big_tokenizer.get_vocab().keys() )[:1_0]
           lowerCAmelCase    :		List[str]				=      " ".join(UpperCamelCase__ )
           lowerCAmelCase    :		Union[str, Any]				=      self.big_tokenizer.encode_plus(UpperCamelCase__    , return_tensors='''pt'''    , return_token_type_ids=UpperCamelCase__ )
           lowerCAmelCase    :		Any				=      self.big_tokenizer.batch_encode_plus(
               [sequence + ''' ''' + sequence]    , return_tensors='''pt'''    , return_token_type_ids=UpperCamelCase__ )

           lowerCAmelCase    :		Optional[int]				=      BigBirdConfig(attention_type='''original_full''' )
           lowerCAmelCase    :		Optional[int]				=      BigBirdModel(UpperCamelCase__ )

           assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

           with torch.no_grad():
                 model(**UpperCamelCase__ )
                 model(**UpperCamelCase__ )






     @slow
     def    lowerCamelCase__   (		self     :    Optional[int] ):
           lowerCAmelCase    :		List[str]				=      BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''' )
           lowerCAmelCase    :		Union[str, Any]				=      tokenizer.decode(tokenizer('''Paris is the [MASK].''' ).input_ids )

           self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]''' )






     @slow
     def    lowerCamelCase__   (		self     :    Optional[int] ):
           lowerCAmelCase    :		str				=      {"input_ids": [[6_5, 3_9_2_8_6, 4_5_8, 3_6_3_3_5, 2_0_0_1, 4_5_6, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 7_7_4_6, 1_7_4_1, 1_1_1_5_7, 3_9_1, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 3_9_6_7, 3_5_4_1_2, 1_1_3, 4_9_3_6, 1_0_9, 3_8_7_0, 2_3_7_7, 1_1_3, 3_0_0_8_4, 4_5_7_2_0, 4_5_8, 1_3_4, 1_7_4_9_6, 1_1_2, 5_0_3, 1_1_6_7_2, 1_1_3, 1_1_8, 1_1_2, 5_6_6_5, 1_3_3_4_7, 3_8_6_8_7, 1_1_2, 1_4_9_6, 3_1_3_8_9, 1_1_2, 3_2_6_8, 4_7_2_6_4, 1_3_4, 9_6_2, 1_1_2, 1_6_3_7_7, 8_0_3_5, 2_3_1_3_0, 4_3_0, 1_2_1_6_9, 1_5_5_1_8, 2_8_5_9_2, 4_5_8, 1_4_6, 4_1_6_9_7, 1_0_9, 3_9_1, 1_2_1_6_9, 1_5_5_1_8, 1_6_6_8_9, 4_5_8, 1_4_6, 4_1_3_5_8, 1_0_9, 4_5_2, 7_2_6, 4_0_3_4, 1_1_1, 7_6_3, 3_5_4_1_2, 5_0_8_2, 3_8_8, 1_9_0_3, 1_1_1, 9_0_5_1, 3_9_1, 2_8_7_0, 4_8_9_1_8, 1_9_0_0, 1_1_2_3, 5_5_0, 9_9_8, 1_1_2, 9_5_8_6, 1_5_9_8_5, 4_5_5, 3_9_1, 4_1_0, 2_2_9_5_5, 3_7_6_3_6, 1_1_4, 6_6], [6_5, 4_4_8, 1_7_4_9_6, 4_1_9, 3_6_6_3, 3_8_5, 7_6_3, 1_1_3, 2_7_5_3_3, 2_8_7_0, 3_2_8_3, 1_3_0_4_3, 1_6_3_9, 2_4_7_1_3, 5_2_3, 6_5_6, 2_4_0_1_3, 1_8_5_5_0, 2_5_2_1, 5_1_7, 2_7_0_1_4, 2_1_2_4_4, 4_2_0, 1_2_1_2, 1_4_6_5, 3_9_1, 9_2_7, 4_8_3_3, 3_8_8, 5_7_8, 1_1_7_8_6, 1_1_4, 6_6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6_5, 4_8_4, 2_1_6_9, 7_6_8_7, 2_1_9_3_2, 1_8_1_4_6, 7_2_6, 3_6_3, 1_7_0_3_2, 3_3_9_1, 1_1_4, 6_6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase__    , model_name='''google/bigbird-roberta-base'''    , revision='''215c99f1600e06f83acce68422f2035b2b5c3510'''    , )





"""simple docstring"""


import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[str]         =  None
				SCREAMING_SNAKE_CASE_   :      Tuple         =  BloomTokenizerFast
				SCREAMING_SNAKE_CASE_   :      str         =  BloomTokenizerFast
				SCREAMING_SNAKE_CASE_   :      int         =  True
				SCREAMING_SNAKE_CASE_   :      Tuple         =  False
				SCREAMING_SNAKE_CASE_   :      int         =  """tokenizer_file"""
				SCREAMING_SNAKE_CASE_   :      List[str]         =  {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}

				def 							lowercase_     (	self							:				List[Any])->   Dict:



						'''simple docstring'''


						super().setUp()
						__lowerCAmelCase:       Optional[Any]          =			BloomTokenizerFast.from_pretrained("bigscience/tokenizer")
						tokenizer.save_pretrained(self.tmpdirname)

				def 							lowercase_     (	self							:				List[Any]     ,  **UpperCamelCase__							:				Union[str, Any])->   Optional[Any]:



						'''simple docstring'''


						kwargs.update(self.special_tokens_map)
						return BloomTokenizerFast.from_pretrained(self.tmpdirname     ,  **UpperCamelCase__)

				def 							lowercase_     (	self							:				Union[str, Any])->   Union[str, Any]:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.get_rust_tokenizer()

						__lowerCAmelCase:       int          =			["The quick brown fox</s>", "jumps over the lazy dog</s>"]
						__lowerCAmelCase:       List[str]          =			[[2_1_7_5, 2_3_7_1_4, 7_3_1_7_3, 1_4_4_2_5_2, 2], [7_7, 1_3_2_6_1_9, 3_4_7_8, 3_6_8, 1_0_9_5_8_6, 3_5_4_3_3, 2]]

						__lowerCAmelCase:       List[str]          =			tokenizer.batch_encode_plus(UpperCamelCase__)["input_ids"]
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

						__lowerCAmelCase:       List[Any]          =			tokenizer.batch_decode(UpperCamelCase__)
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				def 							lowercase_     (	self							:				Optional[Any]     ,  UpperCamelCase__							:				Tuple=6)->   Tuple:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Optional[Any]          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										# tokenizer_r.pad_token = None # Hotfixing padding = None
										# Simple input
										__lowerCAmelCase:       Dict          =			"This is a simple input"
										__lowerCAmelCase:       str          =			["This is a simple input 1", "This is a simple input 2"]
										__lowerCAmelCase:       int          =			("This is a simple input", "This is a pair")
										__lowerCAmelCase:       Union[str, Any]          =			[
										    ("This is a simple input 1", "This is a simple input 2"),
										    ("This is a simple pair 1", "This is a simple pair 2"),
										]

										# Simple input tests
										try:
												tokenizer_r.encode(UpperCamelCase__     ,  max_length=UpperCamelCase__)
												tokenizer_r.encode_plus(UpperCamelCase__     ,  max_length=UpperCamelCase__)

												tokenizer_r.batch_encode_plus(UpperCamelCase__     ,  max_length=UpperCamelCase__)
												tokenizer_r.encode(UpperCamelCase__     ,  max_length=UpperCamelCase__)
												tokenizer_r.batch_encode_plus(UpperCamelCase__     ,  max_length=UpperCamelCase__)
										except ValueError:
												self.fail("Bloom Tokenizer should be able to deal with padding")

										__lowerCAmelCase:       Tuple          =			None  # Hotfixing padding = None
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Simple input
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Simple input
										self.assertRaises(
										    UpperCamelCase__     ,  tokenizer_r.batch_encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length"     ,  )

										# Pair input
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Pair input
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Pair input
										self.assertRaises(
										    UpperCamelCase__     ,  tokenizer_r.batch_encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length"     ,  )

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       Dict          =			self.get_rust_tokenizer()
						__lowerCAmelCase:       List[str]          =			load_dataset("xnli"     ,  "all_languages"     ,  split="test"     ,  streaming=UpperCamelCase__)

						__lowerCAmelCase:       Union[str, Any]          =			next(iter(UpperCamelCase__))["premise"]  # pick up one data
						__lowerCAmelCase:       Any          =			list(sample_data.values())

						__lowerCAmelCase:       int          =			list(map(tokenizer.encode     ,  UpperCamelCase__))
						__lowerCAmelCase:       str          =			[tokenizer.decode(UpperCamelCase__     ,  clean_up_tokenization_spaces=UpperCamelCase__) for x in output_tokens]
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)




				def 							lowercase_     (	self							:				Optional[int])->   str:



						'''simple docstring'''


						self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)     ,  1)
						self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])     ,  1)


