





import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class    __SCREAMING_SNAKE_CASE			( UpperCamelCase_     ,unittest.TestCase					):
       _UpperCAmelCase      :  Optional[Any]     =    GPTaTokenizer
       _UpperCAmelCase      :  List[Any]     =    GPTaTokenizerFast
       _UpperCAmelCase      :  int     =    True
       _UpperCAmelCase      :  str     =    {"add_prefix_space": True}
       _UpperCAmelCase      :  Any     =    False







       def 						__lowerCamelCase    (      self     :	List[str]			)   ->Optional[Any]:
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          lowerCamelCase__   :						Optional[int]					   =							[
              '''l''',
              '''o''',
              '''w''',
              '''e''',
              '''r''',
              '''s''',
              '''t''',
              '''i''',
              '''d''',
              '''n''',
              '''\u0120''',
              '''\u0120l''',
              '''\u0120n''',
              '''\u0120lo''',
              '''\u0120low''',
              '''er''',
              '''\u0120lowest''',
              '''\u0120newer''',
              '''\u0120wider''',
              '''<unk>''',
              '''<|endoftext|>''',
          ]
          lowerCamelCase__   :						Union[str, Any]					   =							dict(zip(UpperCamelCase__		,					range(len(UpperCamelCase__			)			)			)			)
          lowerCamelCase__   :						Any					   =							['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
          lowerCamelCase__   :						Optional[Any]					   =							{'''unk_token''': '''<unk>'''}

          lowerCamelCase__   :						List[Any]					   =							os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES['''vocab_file''']			)
          lowerCamelCase__   :						List[Any]					   =							os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES['''merges_file''']			)
          with open(self.vocab_file		,					'''w'''		,					encoding='''utf-8'''			) as fp:
             fp.write(json.dumps(UpperCamelCase__			) + '''\n'''			)
          with open(self.merges_file		,					'''w'''		,					encoding='''utf-8'''			) as fp:
             fp.write('''\n'''.join(UpperCamelCase__			)			)







       def 						__lowerCamelCase    (      self     :	List[str]		,					**A     :	List[str]			)   ->Tuple:
          kwargs.update(self.special_tokens_map			)
          return GPTaTokenizer.from_pretrained(self.tmpdirname		,					**UpperCamelCase__			)







       def 						__lowerCamelCase    (      self     :	int		,					**A     :	List[str]			)   ->Optional[int]:
          kwargs.update(self.special_tokens_map			)
          return GPTaTokenizerFast.from_pretrained(self.tmpdirname		,					**UpperCamelCase__			)







       def 						__lowerCamelCase    (      self     :	str		,					A     :	int			)   ->List[str]:
          lowerCamelCase__   :						Tuple					   =							'''lower newer'''
          lowerCamelCase__   :						Tuple					   =							'''lower newer'''
          return input_text, output_text







       def 						__lowerCamelCase    (      self     :	int			)   ->int:
          lowerCamelCase__   :						Dict					   =							GPTaTokenizer(self.vocab_file		,					self.merges_file		,					**self.special_tokens_map			)
          lowerCamelCase__   :						List[str]					   =							'''lower newer'''
          lowerCamelCase__   :						Any					   =							['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
          lowerCamelCase__   :						Optional[Any]					   =							tokenizer.tokenize(UpperCamelCase__		,					add_prefix_space=UpperCamelCase__			)
          self.assertListEqual(UpperCamelCase__		,					UpperCamelCase__			)

          lowerCamelCase__   :						List[Any]					   =							tokens + [tokenizer.unk_token]
          lowerCamelCase__   :						List[Any]					   =							[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__			)		,					UpperCamelCase__			)







       def 						__lowerCamelCase    (      self     :	Tuple			)   ->Any:





          if not self.test_rust_tokenizer:
             return

          lowerCamelCase__   :						Optional[Any]					   =							self.get_tokenizer()
          lowerCamelCase__   :						Optional[Any]					   =							self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__			)

          lowerCamelCase__   :						Optional[int]					   =							'''lower newer'''

          # Testing tokenization
          lowerCamelCase__   :						int					   =							tokenizer.tokenize(UpperCamelCase__		,					add_prefix_space=UpperCamelCase__			)
          lowerCamelCase__   :						List[str]					   =							rust_tokenizer.tokenize(UpperCamelCase__			)
          self.assertListEqual(UpperCamelCase__		,					UpperCamelCase__			)

          # Testing conversion to ids without special tokens
          lowerCamelCase__   :						Optional[int]					   =							tokenizer.encode(UpperCamelCase__		,					add_special_tokens=UpperCamelCase__		,					add_prefix_space=UpperCamelCase__			)
          lowerCamelCase__   :						List[str]					   =							rust_tokenizer.encode(UpperCamelCase__		,					add_special_tokens=UpperCamelCase__			)
          self.assertListEqual(UpperCamelCase__		,					UpperCamelCase__			)

          # Testing conversion to ids with special tokens
          lowerCamelCase__   :						Optional[int]					   =							self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__			)
          lowerCamelCase__   :						Any					   =							tokenizer.encode(UpperCamelCase__		,					add_prefix_space=UpperCamelCase__			)
          lowerCamelCase__   :						int					   =							rust_tokenizer.encode(UpperCamelCase__			)
          self.assertListEqual(UpperCamelCase__		,					UpperCamelCase__			)

          # Testing the unknown token
          lowerCamelCase__   :						List[str]					   =							tokens + [rust_tokenizer.unk_token]
          lowerCamelCase__   :						str					   =							[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase__			)		,					UpperCamelCase__			)







       def 						__lowerCamelCase    (      self     :	str		,					*A     :	int		,					**A     :	Optional[Any]			)   ->Dict:
          pass







       def 						__lowerCamelCase    (      self     :	Optional[int]		,					A     :	Any=1_5			)   ->List[str]:





          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"			):
                lowerCamelCase__   :						Any					   =							self.rust_tokenizer_class.from_pretrained(UpperCamelCase__		,					**UpperCamelCase__			)

                # Simple input
                lowerCamelCase__   :						Tuple					   =							'''This is a simple input'''
                lowerCamelCase__   :						Dict					   =							['''This is a simple input 1''', '''This is a simple input 2''']
                lowerCamelCase__   :						Tuple					   =							('''This is a simple input''', '''This is a pair''')
                lowerCamelCase__   :						List[Any]					   =							[
                    ('''This is a simple input 1''', '''This is a simple input 2'''),
                    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                ]

                # Simple input tests
                self.assertRaises(UpperCamelCase__		,					tokenizer_r.encode		,					UpperCamelCase__		,					max_length=UpperCamelCase__		,					padding='''max_length'''			)

                # Simple input
                self.assertRaises(UpperCamelCase__		,					tokenizer_r.encode_plus		,					UpperCamelCase__		,					max_length=UpperCamelCase__		,					padding='''max_length'''			)

                # Simple input
                self.assertRaises(
                    UpperCamelCase__		,					tokenizer_r.batch_encode_plus		,					UpperCamelCase__		,					max_length=UpperCamelCase__		,					padding='''max_length'''		,					)

                # Pair input
                self.assertRaises(UpperCamelCase__		,					tokenizer_r.encode		,					UpperCamelCase__		,					max_length=UpperCamelCase__		,					padding='''max_length'''			)

                # Pair input
                self.assertRaises(UpperCamelCase__		,					tokenizer_r.encode_plus		,					UpperCamelCase__		,					max_length=UpperCamelCase__		,					padding='''max_length'''			)

                # Pair input
                self.assertRaises(
                    UpperCamelCase__		,					tokenizer_r.batch_encode_plus		,					UpperCamelCase__		,					max_length=UpperCamelCase__		,					padding='''max_length'''		,					)







       def 						__lowerCamelCase    (      self     :	Optional[int]			)   ->Union[str, Any]:
          lowerCamelCase__   :						Tuple					   =							GPTaTokenizer.from_pretrained(self.tmpdirname		,					pad_token='''<pad>'''			)

          # Simple input
          lowerCamelCase__   :						int					   =							'''This is a simple input'''
          lowerCamelCase__   :						Any					   =							['''This is a simple input looooooooong''', '''This is a simple input''']
          lowerCamelCase__   :						Optional[int]					   =							('''This is a simple input''', '''This is a pair''')
          lowerCamelCase__   :						List[Any]					   =							[
              ('''This is a simple input loooooong''', '''This is a simple input'''),
              ('''This is a simple pair loooooong''', '''This is a simple pair'''),
          ]

          lowerCamelCase__   :						str					   =							tokenizer.pad_token_id

          lowerCamelCase__   :						Tuple					   =							tokenizer(UpperCamelCase__		,					padding='''max_length'''		,					max_length=3_0		,					return_tensors='''np'''			)
          lowerCamelCase__   :						Dict					   =							tokenizer(UpperCamelCase__		,					padding=UpperCamelCase__		,					truncate=UpperCamelCase__		,					return_tensors='''np'''			)
          lowerCamelCase__   :						Union[str, Any]					   =							tokenizer(*UpperCamelCase__		,					padding='''max_length'''		,					max_length=6_0		,					return_tensors='''np'''			)
          lowerCamelCase__   :						Optional[Any]					   =							tokenizer(UpperCamelCase__		,					padding=UpperCamelCase__		,					truncate=UpperCamelCase__		,					return_tensors='''np'''			)

          # s
          # test single string max_length padding
          self.assertEqual(out_s['''input_ids'''].shape[-1]		,					3_0			)
          self.assertTrue(pad_token_id in out_s['''input_ids''']			)
          self.assertTrue(0 in out_s['''attention_mask''']			)

          # s2
          # test automatic padding
          self.assertEqual(out_sa['''input_ids'''].shape[-1]		,					3_3			)
          # long slice doesn't have padding
          self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]			)
          self.assertFalse(0 in out_sa['''attention_mask'''][0]			)
          # short slice does have padding
          self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]			)
          self.assertTrue(0 in out_sa['''attention_mask'''][1]			)

          # p
          # test single pair max_length padding
          self.assertEqual(out_p['''input_ids'''].shape[-1]		,					6_0			)
          self.assertTrue(pad_token_id in out_p['''input_ids''']			)
          self.assertTrue(0 in out_p['''attention_mask''']			)

          # p2
          # test automatic padding pair
          self.assertEqual(out_pa['''input_ids'''].shape[-1]		,					5_2			)
          # long slice pair doesn't have padding
          self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]			)
          self.assertFalse(0 in out_pa['''attention_mask'''][0]			)
          # short slice pair does have padding
          self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]			)
          self.assertTrue(0 in out_pa['''attention_mask'''][1]			)







       def 						__lowerCamelCase    (      self     :	Dict			)   ->str:
          lowerCamelCase__   :						Optional[int]					   =							'''$$$'''
          lowerCamelCase__   :						List[str]					   =							GPTaTokenizer.from_pretrained(self.tmpdirname		,					bos_token=UpperCamelCase__		,					add_bos_token=UpperCamelCase__			)

          lowerCamelCase__   :						Optional[Any]					   =							'''This is a simple input'''
          lowerCamelCase__   :						Optional[Any]					   =							['''This is a simple input 1''', '''This is a simple input 2''']

          lowerCamelCase__   :						List[Any]					   =							tokenizer.bos_token_id

          lowerCamelCase__   :						Dict					   =							tokenizer(UpperCamelCase__			)
          lowerCamelCase__   :						Tuple					   =							tokenizer(UpperCamelCase__			)

          self.assertEqual(out_s.input_ids[0]		,					UpperCamelCase__			)
          self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

          lowerCamelCase__   :						Union[str, Any]					   =							tokenizer.decode(out_s.input_ids			)
          lowerCamelCase__   :						Optional[Any]					   =							tokenizer.batch_decode(out_sa.input_ids			)

          self.assertEqual(decode_s.split()[0]		,					UpperCamelCase__			)
          self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)







       def 						__lowerCamelCase    (      self     :	Dict			)   ->List[str]:
          pass







       def 						__lowerCamelCase    (      self     :	List[Any]			)   ->Dict:
          lowerCamelCase__   :						List[str]					   =							[self.get_tokenizer(do_lower_case=UpperCamelCase__		,					add_bos_token=UpperCamelCase__			)]
          for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"			):
                lowerCamelCase__   :						Dict					   =							'''Encode this.'''
                lowerCamelCase__   :						Tuple					   =							'''This one too please.'''
                lowerCamelCase__   :						Any					   =							tokenizer.encode(UpperCamelCase__		,					add_special_tokens=UpperCamelCase__			)
                encoded_sequence += tokenizer.encode(UpperCamelCase__		,					add_special_tokens=UpperCamelCase__			)
                lowerCamelCase__   :						str					   =							tokenizer.encode_plus(
                    UpperCamelCase__		,					UpperCamelCase__		,					add_special_tokens=UpperCamelCase__		,					return_special_tokens_mask=UpperCamelCase__		,					)
                lowerCamelCase__   :						List[Any]					   =							encoded_sequence_dict['''input_ids''']
                lowerCamelCase__   :						Dict					   =							encoded_sequence_dict['''special_tokens_mask''']
                self.assertEqual(len(UpperCamelCase__			)		,					len(UpperCamelCase__			)			)

                lowerCamelCase__   :						Any					   =							[
                    (x if not special_tokens_mask[i] else None) for i, x in enumerate(UpperCamelCase__			)
                ]
                lowerCamelCase__   :						List[Any]					   =							[x for x in filtered_sequence if x is not None]
                self.assertEqual(UpperCamelCase__		,					UpperCamelCase__			)


@require_tokenizers
class    __SCREAMING_SNAKE_CASE			( unittest.TestCase					):







       def 						__lowerCamelCase    (      self     :	Optional[int]			)   ->Optional[Any]:
          lowerCamelCase__   :						str					   =							AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,					from_slow=UpperCamelCase__			)
          lowerCamelCase__   :						Any					   =							'''A photo of a cat'''

          lowerCamelCase__   :						Dict					   =							tokenizer.encode(
              UpperCamelCase__		,					)
          self.assertEqual(UpperCamelCase__		,					[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)
          tokenizer.save_pretrained('''test_opt'''			)

          lowerCamelCase__   :						Dict					   =							AutoTokenizer.from_pretrained('''./test_opt'''			)
          lowerCamelCase__   :						Optional[int]					   =							tokenizer.encode(
              UpperCamelCase__		,					)
          self.assertEqual(UpperCamelCase__		,					[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)







       def 						__lowerCamelCase    (      self     :	List[Any]			)   ->Any:
          lowerCamelCase__   :						Optional[int]					   =							AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,					use_slow=UpperCamelCase__			)
          lowerCamelCase__   :						Union[str, Any]					   =							'''A photo of a cat'''

          lowerCamelCase__   :						Union[str, Any]					   =							tokenizer.encode(
              UpperCamelCase__		,					)
          # Same as above
          self.assertEqual(UpperCamelCase__		,					[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)







       @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''			)
       def 						__lowerCamelCase    (      self     :	List[str]			)   ->int:
          lowerCamelCase__   :						Any					   =							AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,					from_slow=UpperCamelCase__			)

          lowerCamelCase__   :						Optional[Any]					   =							'''bos'''
          lowerCamelCase__   :						List[str]					   =							tokenizer.get_vocab()['''bos''']

          lowerCamelCase__   :						Optional[int]					   =							'''A photo of a cat'''
          lowerCamelCase__   :						Union[str, Any]					   =							tokenizer.encode(
              UpperCamelCase__		,					)
          # We changed the bos token
          self.assertEqual(UpperCamelCase__		,					[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)
          tokenizer.save_pretrained('''./tok'''			)
          lowerCamelCase__   :						List[Any]					   =							AutoTokenizer.from_pretrained('''./tok'''			)
          self.assertTrue(tokenizer.is_fast			)
          lowerCamelCase__   :						Dict					   =							tokenizer.encode(
              UpperCamelCase__		,					)
          self.assertEqual(UpperCamelCase__		,					[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)




import math
def       A	(    _lowercase						):
						return math.sqrt(_lowercase						) * math.sqrt(_lowercase						) == num
def       A	(    _lowercase						):
						SCREAMING_SNAKE_CASE							:    Dict							=							0
						SCREAMING_SNAKE_CASE							:    Tuple							=							n
						while left <= right:
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							(left + right) // 2
												if mid**2 == n:
																		return True
												elif mid**2 > n:
																		SCREAMING_SNAKE_CASE							:    Optional[Any]							=							mid - 1
												else:
																		SCREAMING_SNAKE_CASE							:    Optional[int]							=							mid + 1
						return False


if __name__ == "__main__":
					import doctest

					doctest.testmod()





