
'''simple docstring'''







import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 						__UpperCAmelCase      (					_lowerCamelCase						,   unittest.TestCase						):
							__lowercase      		=       GPTaTokenizer
							__lowercase      		=       GPTaTokenizerFast
							__lowercase      		=       True
							__lowercase      		=       {'''add_prefix_space''': True}
							__lowercase      		=       False


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								_snake_case   						=		[
								    '''l''',
								    '''o''',
								    '''w''',
								    '''e''',
								    '''r''',
								    '''s''',
								    '''t''',
								    '''i''',
								    '''d''',
								    '''n''',
								    '''\u0120''',
								    '''\u0120l''',
								    '''\u0120n''',
								    '''\u0120lo''',
								    '''\u0120low''',
								    '''er''',
								    '''\u0120lowest''',
								    '''\u0120newer''',
								    '''\u0120wider''',
								    '''<unk>''',
								    '''<|endoftext|>''',
								]
								_snake_case   						=		dict(zip(SCREAMING_SNAKE_CASE__    ,   range(len(SCREAMING_SNAKE_CASE__  )  )  )  )
								_snake_case   						=		['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
								_snake_case   						=		{'''unk_token''': '''<unk>'''}

								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['vocab_file']  )
								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['merges_file']  )
								with open(self.vocab_file    ,   'w'    ,   encoding='utf-8'  ) as fp:
									fp.write(json.dumps(SCREAMING_SNAKE_CASE__  ) + '\n'  )
								with open(self.merges_file    ,   'w'    ,   encoding='utf-8'  ) as fp:
									fp.write('\n'.join(SCREAMING_SNAKE_CASE__  )  )


							def  lowerCamelCase				(       self    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								kwargs.update(self.special_tokens_map  )
								return GPTaTokenizer.from_pretrained(self.tmpdirname    ,   **SCREAMING_SNAKE_CASE__  )


							def  lowerCamelCase				(       self    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								kwargs.update(self.special_tokens_map  )
								return GPTaTokenizerFast.from_pretrained(self.tmpdirname    ,   **SCREAMING_SNAKE_CASE__  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		'''lower newer'''
								_snake_case   						=		'''lower newer'''
								return input_text, output_text


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		GPTaTokenizer(self.vocab_file    ,   self.merges_file    ,   **self.special_tokens_map  )
								_snake_case   						=		'''lower newer'''
								_snake_case   						=		['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
								_snake_case   						=		tokenizer.tokenize(SCREAMING_SNAKE_CASE__    ,   add_prefix_space=SCREAMING_SNAKE_CASE__  )
								self.assertListEqual(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )

								_snake_case   						=		tokens + [tokenizer.unk_token]
								_snake_case   						=		[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )    ,   SCREAMING_SNAKE_CASE__  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								if not self.test_rust_tokenizer:
									return

								_snake_case   						=		self.get_tokenizer()
								_snake_case   						=		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__  )

								_snake_case   						=		'''lower newer'''

								# Testing tokenization
								_snake_case   						=		tokenizer.tokenize(SCREAMING_SNAKE_CASE__    ,   add_prefix_space=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
								self.assertListEqual(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )

								# Testing conversion to ids without special tokens
								_snake_case   						=		tokenizer.encode(SCREAMING_SNAKE_CASE__    ,   add_special_tokens=SCREAMING_SNAKE_CASE__    ,   add_prefix_space=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		rust_tokenizer.encode(SCREAMING_SNAKE_CASE__    ,   add_special_tokens=SCREAMING_SNAKE_CASE__  )
								self.assertListEqual(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )

								# Testing conversion to ids with special tokens
								_snake_case   						=		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		tokenizer.encode(SCREAMING_SNAKE_CASE__    ,   add_prefix_space=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		rust_tokenizer.encode(SCREAMING_SNAKE_CASE__  )
								self.assertListEqual(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )

								# Testing the unknown token
								_snake_case   						=		tokens + [rust_tokenizer.unk_token]
								_snake_case   						=		[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )    ,   SCREAMING_SNAKE_CASE__  )


							def  lowerCamelCase				(       self    ,   *lowerCAmelCase_    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								pass


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_=15  ):





								"""simple docstring"""






								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
										_snake_case   						=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__    ,   **SCREAMING_SNAKE_CASE__  )

										# Simple input
										_snake_case   						=		'''This is a simple input'''
										_snake_case   						=		['''This is a simple input 1''', '''This is a simple input 2''']
										_snake_case   						=		('''This is a simple input''', '''This is a pair''')
										_snake_case   						=		[
										    ('''This is a simple input 1''', '''This is a simple input 2'''),
										    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
										]

										# Simple input tests
										self.assertRaises(SCREAMING_SNAKE_CASE__    ,   tokenizer_r.encode    ,   SCREAMING_SNAKE_CASE__    ,   max_length=SCREAMING_SNAKE_CASE__    ,   padding='max_length'  )

										# Simple input
										self.assertRaises(SCREAMING_SNAKE_CASE__    ,   tokenizer_r.encode_plus    ,   SCREAMING_SNAKE_CASE__    ,   max_length=SCREAMING_SNAKE_CASE__    ,   padding='max_length'  )

										# Simple input
										self.assertRaises(
										    SCREAMING_SNAKE_CASE__    ,   tokenizer_r.batch_encode_plus    ,   SCREAMING_SNAKE_CASE__    ,   max_length=SCREAMING_SNAKE_CASE__    ,   padding='max_length'    ,   )

										# Pair input
										self.assertRaises(SCREAMING_SNAKE_CASE__    ,   tokenizer_r.encode    ,   SCREAMING_SNAKE_CASE__    ,   max_length=SCREAMING_SNAKE_CASE__    ,   padding='max_length'  )

										# Pair input
										self.assertRaises(SCREAMING_SNAKE_CASE__    ,   tokenizer_r.encode_plus    ,   SCREAMING_SNAKE_CASE__    ,   max_length=SCREAMING_SNAKE_CASE__    ,   padding='max_length'  )

										# Pair input
										self.assertRaises(
										    SCREAMING_SNAKE_CASE__    ,   tokenizer_r.batch_encode_plus    ,   SCREAMING_SNAKE_CASE__    ,   max_length=SCREAMING_SNAKE_CASE__    ,   padding='max_length'    ,   )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		GPTaTokenizer.from_pretrained(self.tmpdirname    ,   pad_token='<pad>'  )

								# Simple input
								_snake_case   						=		'''This is a simple input'''
								_snake_case   						=		['''This is a simple input looooooooong''', '''This is a simple input''']
								_snake_case   						=		('''This is a simple input''', '''This is a pair''')
								_snake_case   						=		[
								    ('''This is a simple input loooooong''', '''This is a simple input'''),
								    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
								]

								_snake_case   						=		tokenizer.pad_token_id

								_snake_case   						=		tokenizer(SCREAMING_SNAKE_CASE__    ,   padding='max_length'    ,   max_length=30    ,   return_tensors='np'  )
								_snake_case   						=		tokenizer(SCREAMING_SNAKE_CASE__    ,   padding=SCREAMING_SNAKE_CASE__    ,   truncate=SCREAMING_SNAKE_CASE__    ,   return_tensors='np'  )
								_snake_case   						=		tokenizer(*SCREAMING_SNAKE_CASE__    ,   padding='max_length'    ,   max_length=60    ,   return_tensors='np'  )
								_snake_case   						=		tokenizer(SCREAMING_SNAKE_CASE__    ,   padding=SCREAMING_SNAKE_CASE__    ,   truncate=SCREAMING_SNAKE_CASE__    ,   return_tensors='np'  )

								# s
								# test single string max_length padding
								self.assertEqual(out_s['input_ids'].shape[-1]    ,   30  )
								self.assertTrue(pad_token_id in out_s['input_ids']  )
								self.assertTrue(0 in out_s['attention_mask']  )

								# s2
								# test automatic padding
								self.assertEqual(out_sa['input_ids'].shape[-1]    ,   33  )
								# long slice doesn't have padding
								self.assertFalse(pad_token_id in out_sa['input_ids'][0]  )
								self.assertFalse(0 in out_sa['attention_mask'][0]  )
								# short slice does have padding
								self.assertTrue(pad_token_id in out_sa['input_ids'][1]  )
								self.assertTrue(0 in out_sa['attention_mask'][1]  )

								# p
								# test single pair max_length padding
								self.assertEqual(out_p['input_ids'].shape[-1]    ,   60  )
								self.assertTrue(pad_token_id in out_p['input_ids']  )
								self.assertTrue(0 in out_p['attention_mask']  )

								# p2
								# test automatic padding pair
								self.assertEqual(out_pa['input_ids'].shape[-1]    ,   52  )
								# long slice pair doesn't have padding
								self.assertFalse(pad_token_id in out_pa['input_ids'][0]  )
								self.assertFalse(0 in out_pa['attention_mask'][0]  )
								# short slice pair does have padding
								self.assertTrue(pad_token_id in out_pa['input_ids'][1]  )
								self.assertTrue(0 in out_pa['attention_mask'][1]  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'''$$$'''
								_snake_case   						=		GPTaTokenizer.from_pretrained(self.tmpdirname    ,   bos_token=SCREAMING_SNAKE_CASE__    ,   add_bos_token=SCREAMING_SNAKE_CASE__  )

								_snake_case   						=		'''This is a simple input'''
								_snake_case   						=		['''This is a simple input 1''', '''This is a simple input 2''']

								_snake_case   						=		tokenizer.bos_token_id

								_snake_case   						=		tokenizer(SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		tokenizer(SCREAMING_SNAKE_CASE__  )

								self.assertEqual(out_s.input_ids[0]    ,   SCREAMING_SNAKE_CASE__  )
								self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids  )  )

								_snake_case   						=		tokenizer.decode(out_s.input_ids  )
								_snake_case   						=		tokenizer.batch_decode(out_sa.input_ids  )

								self.assertEqual(decode_s.split()[0]    ,   SCREAMING_SNAKE_CASE__  )
								self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa  )  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass




							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		[self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE__    ,   add_bos_token=SCREAMING_SNAKE_CASE__  )]
								for tokenizer in tokenizers:
									with self.subTest(F'{tokenizer.__class__.__name__}'  ):
										_snake_case   						=		'''Encode this.'''
										_snake_case   						=		'''This one too please.'''
										_snake_case   						=		tokenizer.encode(SCREAMING_SNAKE_CASE__    ,   add_special_tokens=SCREAMING_SNAKE_CASE__  )
										encoded_sequence += tokenizer.encode(SCREAMING_SNAKE_CASE__    ,   add_special_tokens=SCREAMING_SNAKE_CASE__  )
										_snake_case   						=		tokenizer.encode_plus(
										    SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__    ,   add_special_tokens=SCREAMING_SNAKE_CASE__    ,   return_special_tokens_mask=SCREAMING_SNAKE_CASE__    ,   )
										_snake_case   						=		encoded_sequence_dict['''input_ids''']
										_snake_case   						=		encoded_sequence_dict['''special_tokens_mask''']
										self.assertEqual(len(SCREAMING_SNAKE_CASE__  )    ,   len(SCREAMING_SNAKE_CASE__  )  )

										_snake_case   						=		[
										    (x if not special_tokens_mask[i] else None) for i, x in enumerate(SCREAMING_SNAKE_CASE__  )
										]
										_snake_case   						=		[x for x in filtered_sequence if x is not None]
										self.assertEqual(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )


@require_tokenizers
class 						__UpperCAmelCase      (					unittest.TestCase						):


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		AutoTokenizer.from_pretrained('facebook/opt-350m'    ,   from_slow=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		'''A photo of a cat'''

								_snake_case   						=		tokenizer.encode(
								    SCREAMING_SNAKE_CASE__    ,   )
								self.assertEqual(SCREAMING_SNAKE_CASE__    ,   [2, 2_50, 13_45, 9, 10, 47_58]  )
								tokenizer.save_pretrained('test_opt'  )

								_snake_case   						=		AutoTokenizer.from_pretrained('./test_opt'  )
								_snake_case   						=		tokenizer.encode(
								    SCREAMING_SNAKE_CASE__    ,   )
								self.assertEqual(SCREAMING_SNAKE_CASE__    ,   [2, 2_50, 13_45, 9, 10, 47_58]  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		AutoTokenizer.from_pretrained('facebook/opt-350m'    ,   use_slow=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		'''A photo of a cat'''

								_snake_case   						=		tokenizer.encode(
								    SCREAMING_SNAKE_CASE__    ,   )
								# Same as above
								self.assertEqual(SCREAMING_SNAKE_CASE__    ,   [2, 2_50, 13_45, 9, 10, 47_58]  )




							@unittest.skip('This test is failing because of a bug in the fast tokenizer'  )
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		AutoTokenizer.from_pretrained('facebook/opt-350m'    ,   from_slow=SCREAMING_SNAKE_CASE__  )

								_snake_case   						=		'''bos'''
								_snake_case   						=		tokenizer.get_vocab()['''bos''']

								_snake_case   						=		'''A photo of a cat'''
								_snake_case   						=		tokenizer.encode(
								    SCREAMING_SNAKE_CASE__    ,   )
								# We changed the bos token
								self.assertEqual(SCREAMING_SNAKE_CASE__    ,   [3_19_57, 2_50, 13_45, 9, 10, 47_58]  )
								tokenizer.save_pretrained('./tok'  )
								_snake_case   						=		AutoTokenizer.from_pretrained('./tok'  )
								self.assertTrue(tokenizer.is_fast  )
								_snake_case   						=		tokenizer.encode(
								    SCREAMING_SNAKE_CASE__    ,   )
								self.assertEqual(SCREAMING_SNAKE_CASE__    ,   [3_19_57, 2_50, 13_45, 9, 10, 47_58]  )








"""simple docstring"""
from typing import Any



def      __lowerCamelCase   (							a_     :		list							)			->      list[Any]:
      if not input_list:
            return []
      __SCREAMING_SNAKE_CASE   :int						=  [input_list.count(a_							) for value in input_list]
      __SCREAMING_SNAKE_CASE   :str						=  max(a_							)  # Gets the maximum count in the input list.
      # Gets values of modes
      return sorted({input_list[i] for i, value in enumerate(a_							) if value == y}							)


if __name__ == "__main__":
  import doctest

  doctest.testmod()