





import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase						,      unittest.TestCase  ):
      A_						:				Union[str, Any]							=    GPTaTokenizer
      A_						:				List[Any]							=    GPTaTokenizerFast
      A_						:				Dict							=    True
      A_						:				List[str]							=    {'add_prefix_space': True}
      A_						:				List[str]							=    False


      def    a (self       :      Any      ):





         """simple docstring"""



         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         __snake_case							       =   [
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''\u0120''',
             '''\u0120l''',
             '''\u0120n''',
             '''\u0120lo''',
             '''\u0120low''',
             '''er''',
             '''\u0120lowest''',
             '''\u0120newer''',
             '''\u0120wider''',
             '''<unk>''',
             '''<|endoftext|>''',
         ]
         __snake_case							       =   dict(zip(a__				,			range(len(a__      )      )      )      )
         __snake_case							       =   ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
         __snake_case							       =   {'''unk_token''': '''<unk>'''}

         __snake_case							       =   os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES['''vocab_file''']      )
         __snake_case							       =   os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES['''merges_file''']      )
         with open(self.vocab_file				,			'''w'''				,			encoding='''utf-8'''      ) as fp:
            fp.write(json.dumps(a__      ) + '''\n'''      )
         with open(self.merges_file				,			'''w'''				,			encoding='''utf-8'''      ) as fp:
            fp.write('''\n'''.join(a__      )      )


      def    a (self       :      List[str]				,			**a__       :      List[Any]      ):





         """simple docstring"""



         kwargs.update(self.special_tokens_map      )
         return GPTaTokenizer.from_pretrained(self.tmpdirname				,			**a__      )


      def    a (self       :      Dict				,			**a__       :      Union[str, Any]      ):





         """simple docstring"""



         kwargs.update(self.special_tokens_map      )
         return GPTaTokenizerFast.from_pretrained(self.tmpdirname				,			**a__      )


      def    a (self       :      int				,			a__       :      Optional[int]      ):





         """simple docstring"""



         __snake_case							       =   '''lower newer'''
         __snake_case							       =   '''lower newer'''
         return input_text, output_text


      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         __snake_case							       =   GPTaTokenizer(self.vocab_file				,			self.merges_file				,			**self.special_tokens_map      )
         __snake_case							       =   '''lower newer'''
         __snake_case							       =   ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
         __snake_case							       =   tokenizer.tokenize(a__				,			add_prefix_space=a__      )
         self.assertListEqual(a__				,			a__      )

         __snake_case							       =   tokens + [tokenizer.unk_token]
         __snake_case							       =   [14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )				,			a__      )


      def    a (self       :      int      ):





         """simple docstring"""



         if not self.test_rust_tokenizer:
            return

         __snake_case							       =   self.get_tokenizer()
         __snake_case							       =   self.get_rust_tokenizer(add_prefix_space=a__      )

         __snake_case							       =   '''lower newer'''

         # Testing tokenization
         __snake_case							       =   tokenizer.tokenize(a__				,			add_prefix_space=a__      )
         __snake_case							       =   rust_tokenizer.tokenize(a__      )
         self.assertListEqual(a__				,			a__      )

         # Testing conversion to ids without special tokens
         __snake_case							       =   tokenizer.encode(a__				,			add_special_tokens=a__				,			add_prefix_space=a__      )
         __snake_case							       =   rust_tokenizer.encode(a__				,			add_special_tokens=a__      )
         self.assertListEqual(a__				,			a__      )

         # Testing conversion to ids with special tokens
         __snake_case							       =   self.get_rust_tokenizer(add_prefix_space=a__      )
         __snake_case							       =   tokenizer.encode(a__				,			add_prefix_space=a__      )
         __snake_case							       =   rust_tokenizer.encode(a__      )
         self.assertListEqual(a__				,			a__      )

         # Testing the unknown token
         __snake_case							       =   tokens + [rust_tokenizer.unk_token]
         __snake_case							       =   [14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a__      )				,			a__      )


      def    a (self       :      Tuple				,			*a__       :      Optional[int]				,			**a__       :      Optional[Any]      ):





         """simple docstring"""



         pass


      def    a (self       :      List[str]				,			a__       :      Any=15      ):





         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
               __snake_case							       =   self.rust_tokenizer_class.from_pretrained(a__				,			**a__      )

               # Simple input
               __snake_case							       =   '''This is a simple input'''
               __snake_case							       =   ['''This is a simple input 1''', '''This is a simple input 2''']
               __snake_case							       =   ('''This is a simple input''', '''This is a pair''')
               __snake_case							       =   [
                   ('''This is a simple input 1''', '''This is a simple input 2'''),
                   ('''This is a simple pair 1''', '''This is a simple pair 2'''),
               ]

               # Simple input tests
               self.assertRaises(a__				,			tokenizer_r.encode				,			a__				,			max_length=a__				,			padding='''max_length'''      )

               # Simple input
               self.assertRaises(a__				,			tokenizer_r.encode_plus				,			a__				,			max_length=a__				,			padding='''max_length'''      )

               # Simple input
               self.assertRaises(
                   a__				,			tokenizer_r.batch_encode_plus				,			a__				,			max_length=a__				,			padding='''max_length'''				,			)

               # Pair input
               self.assertRaises(a__				,			tokenizer_r.encode				,			a__				,			max_length=a__				,			padding='''max_length'''      )

               # Pair input
               self.assertRaises(a__				,			tokenizer_r.encode_plus				,			a__				,			max_length=a__				,			padding='''max_length'''      )

               # Pair input
               self.assertRaises(
                   a__				,			tokenizer_r.batch_encode_plus				,			a__				,			max_length=a__				,			padding='''max_length'''				,			)


      def    a (self       :      List[str]      ):





         """simple docstring"""



         __snake_case							       =   GPTaTokenizer.from_pretrained(self.tmpdirname				,			pad_token='''<pad>'''      )

         # Simple input
         __snake_case							       =   '''This is a simple input'''
         __snake_case							       =   ['''This is a simple input looooooooong''', '''This is a simple input''']
         __snake_case							       =   ('''This is a simple input''', '''This is a pair''')
         __snake_case							       =   [
             ('''This is a simple input loooooong''', '''This is a simple input'''),
             ('''This is a simple pair loooooong''', '''This is a simple pair'''),
         ]

         __snake_case							       =   tokenizer.pad_token_id

         __snake_case							       =   tokenizer(a__				,			padding='''max_length'''				,			max_length=30				,			return_tensors='''np'''      )
         __snake_case							       =   tokenizer(a__				,			padding=a__				,			truncate=a__				,			return_tensors='''np'''      )
         __snake_case							       =   tokenizer(*a__				,			padding='''max_length'''				,			max_length=60				,			return_tensors='''np'''      )
         __snake_case							       =   tokenizer(a__				,			padding=a__				,			truncate=a__				,			return_tensors='''np'''      )

         # s
         # test single string max_length padding
         self.assertEqual(out_s['''input_ids'''].shape[-1]				,			30      )
         self.assertTrue(pad_token_id in out_s['''input_ids''']      )
         self.assertTrue(0 in out_s['''attention_mask''']      )

         # s2
         # test automatic padding
         self.assertEqual(out_sa['''input_ids'''].shape[-1]				,			33      )
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]      )
         self.assertFalse(0 in out_sa['''attention_mask'''][0]      )
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]      )
         self.assertTrue(0 in out_sa['''attention_mask'''][1]      )

         # p
         # test single pair max_length padding
         self.assertEqual(out_p['''input_ids'''].shape[-1]				,			60      )
         self.assertTrue(pad_token_id in out_p['''input_ids''']      )
         self.assertTrue(0 in out_p['''attention_mask''']      )

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa['''input_ids'''].shape[-1]				,			52      )
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]      )
         self.assertFalse(0 in out_pa['''attention_mask'''][0]      )
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]      )
         self.assertTrue(0 in out_pa['''attention_mask'''][1]      )


      def    a (self       :      Optional[Any]      ):





         """simple docstring"""



         __snake_case							       =   '''$$$'''
         __snake_case							       =   GPTaTokenizer.from_pretrained(self.tmpdirname				,			bos_token=a__				,			add_bos_token=a__      )

         __snake_case							       =   '''This is a simple input'''
         __snake_case							       =   ['''This is a simple input 1''', '''This is a simple input 2''']

         __snake_case							       =   tokenizer.bos_token_id

         __snake_case							       =   tokenizer(a__      )
         __snake_case							       =   tokenizer(a__      )

         self.assertEqual(out_s.input_ids[0]				,			a__      )
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

         __snake_case							       =   tokenizer.decode(out_s.input_ids      )
         __snake_case							       =   tokenizer.batch_decode(out_sa.input_ids      )

         self.assertEqual(decode_s.split()[0]				,			a__      )
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )


      def    a (self       :      List[Any]      ):





         """simple docstring"""



         pass


      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   [self.get_tokenizer(do_lower_case=a__				,			add_bos_token=a__      )]
         for tokenizer in tokenizers:
            with self.subTest(f"""{tokenizer.__class__.__name__}"""      ):
               __snake_case							       =   '''Encode this.'''
               __snake_case							       =   '''This one too please.'''
               __snake_case							       =   tokenizer.encode(a__				,			add_special_tokens=a__      )
               encoded_sequence += tokenizer.encode(a__				,			add_special_tokens=a__      )
               __snake_case							       =   tokenizer.encode_plus(
                   a__				,			a__				,			add_special_tokens=a__				,			return_special_tokens_mask=a__				,			)
               __snake_case							       =   encoded_sequence_dict['''input_ids''']
               __snake_case							       =   encoded_sequence_dict['''special_tokens_mask''']
               self.assertEqual(len(a__      )				,			len(a__      )      )

               __snake_case							       =   [
                   (x if not special_tokens_mask[i] else None) for i, x in enumerate(a__      )
               ]
               __snake_case							       =   [x for x in filtered_sequence if x is not None]
               self.assertEqual(a__				,			a__      )






@require_tokenizers
class 		SCREAMING_SNAKE_CASE__				(       unittest.TestCase  ):


      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         __snake_case							       =   AutoTokenizer.from_pretrained('''facebook/opt-350m'''				,			from_slow=a__      )
         __snake_case							       =   '''A photo of a cat'''

         __snake_case							       =   tokenizer.encode(
             a__				,			)
         self.assertEqual(a__				,			[2, 250, 1345, 9, 10, 4758]      )
         tokenizer.save_pretrained('''test_opt'''      )

         __snake_case							       =   AutoTokenizer.from_pretrained('''./test_opt'''      )
         __snake_case							       =   tokenizer.encode(
             a__				,			)
         self.assertEqual(a__				,			[2, 250, 1345, 9, 10, 4758]      )


      def    a (self       :      List[str]      ):





         """simple docstring"""



         __snake_case							       =   AutoTokenizer.from_pretrained('''facebook/opt-350m'''				,			use_slow=a__      )
         __snake_case							       =   '''A photo of a cat'''

         __snake_case							       =   tokenizer.encode(
             a__				,			)
         # Same as above
         self.assertEqual(a__				,			[2, 250, 1345, 9, 10, 4758]      )


      @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''      )
      def    a (self       :      Tuple      ):





         """simple docstring"""



         __snake_case							       =   AutoTokenizer.from_pretrained('''facebook/opt-350m'''				,			from_slow=a__      )

         __snake_case							       =   '''bos'''
         __snake_case							       =   tokenizer.get_vocab()['''bos''']

         __snake_case							       =   '''A photo of a cat'''
         __snake_case							       =   tokenizer.encode(
             a__				,			)
         # We changed the bos token
         self.assertEqual(a__				,			[3_1957, 250, 1345, 9, 10, 4758]      )
         tokenizer.save_pretrained('''./tok'''      )
         __snake_case							       =   AutoTokenizer.from_pretrained('''./tok'''      )
         self.assertTrue(tokenizer.is_fast      )
         __snake_case							       =   tokenizer.encode(
             a__				,			)
         self.assertEqual(a__				,			[3_1957, 250, 1345, 9, 10, 4758]      )


'''simple docstring'''




import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase						: Optional[Any]						=						logging.get_logger(__name__)

_lowercase						: Tuple						=						{
    "BAAI/AltCLIP": "https://huggingface.co/BAAI/AltCLIP/resolve/main/config.json",
    # See all AltCLIP models at https://huggingface.co/models?filter=altclip
}





class     __magic_name__						(    _UpperCAmelCase):
			UpperCamelCase__				=       '''altclip_text_model'''







			def __init__(	self				:  Union[str, Any]				,			lowercase_				:  str=250002				,			lowercase_				:  Union[str, Any]=1024				,			lowercase_				:  Any=24				,			lowercase_				:  Union[str, Any]=16				,			lowercase_				:  Any=4096				,			lowercase_				:  Union[str, Any]="gelu"				,			lowercase_				:  Union[str, Any]=0.1				,			lowercase_				:  Union[str, Any]=0.1				,			lowercase_				:  int=514				,			lowercase_				:  Union[str, Any]=1				,			lowercase_				:  List[str]=0.02				,			lowercase_				:  Optional[int]=0.02				,			lowercase_				:  str=1E-05				,			lowercase_				:  List[str]=1				,			lowercase_				:  List[Any]=0				,			lowercase_				:  Dict=2				,			lowercase_				:  Union[str, Any]="absolute"				,			lowercase_				:  Any=True				,			lowercase_				:  Union[str, Any]=768				,			**lowercase_				:  Any				,			):
										super().__init__(pad_token_id=lowercase_				,			bos_token_id=lowercase_				,			eos_token_id=lowercase_				,			**lowercase_    )

										lowercase_					:			Union[str, Any]										=       vocab_size
										lowercase_					:			str										=       hidden_size
										lowercase_					:			Optional[Any]										=       num_hidden_layers
										lowercase_					:			int										=       num_attention_heads
										lowercase_					:			str										=       hidden_act
										lowercase_					:			List[str]										=       intermediate_size
										lowercase_					:			Optional[int]										=       hidden_dropout_prob
										lowercase_					:			str										=       attention_probs_dropout_prob
										lowercase_					:			str										=       max_position_embeddings
										lowercase_					:			List[str]										=       type_vocab_size
										lowercase_					:			Union[str, Any]										=       initializer_range
										lowercase_					:			List[Any]										=       initializer_factor
										lowercase_					:			str										=       layer_norm_eps
										lowercase_					:			Tuple										=       position_embedding_type
										lowercase_					:			List[Any]										=       use_cache
										lowercase_					:			Tuple										=       project_dim





class     __magic_name__						(    _UpperCAmelCase):
			UpperCamelCase__				=       '''altclip_vision_model'''







			def __init__(	self				:  Dict				,			lowercase_				:  Any=768				,			lowercase_				:  Dict=3072				,			lowercase_				:  Optional[Any]=512				,			lowercase_				:  Dict=12				,			lowercase_				:  Optional[int]=12				,			lowercase_				:  Optional[Any]=3				,			lowercase_				:  str=224				,			lowercase_				:  List[Any]=32				,			lowercase_				:  Union[str, Any]="quick_gelu"				,			lowercase_				:  Dict=1E-5				,			lowercase_				:  Optional[int]=0.0				,			lowercase_				:  Optional[Any]=0.02				,			lowercase_				:  Optional[Any]=1.0				,			**lowercase_				:  Dict				,			):
										super().__init__(**lowercase_    )

										lowercase_					:			Tuple										=       hidden_size
										lowercase_					:			Optional[Any]										=       intermediate_size
										lowercase_					:			Optional[Any]										=       projection_dim
										lowercase_					:			Tuple										=       num_hidden_layers
										lowercase_					:			List[Any]										=       num_attention_heads
										lowercase_					:			Any										=       num_channels
										lowercase_					:			Any										=       patch_size
										lowercase_					:			Dict										=       image_size
										lowercase_					:			Optional[Any]										=       initializer_range
										lowercase_					:			str										=       initializer_factor
										lowercase_					:			Any										=       attention_dropout
										lowercase_					:			Optional[int]										=       layer_norm_eps
										lowercase_					:			int										=       hidden_act







			@classmethod
			def 	SCREAMING_SNAKE_CASE_     (	cls				:  Optional[int]				,			lowercase_				:  Union[str, os.PathLike]				,			**lowercase_				:  Any    ):
										cls._set_token_in_kwargs(lowercase_    )

										lowercase_						,						lowercase_					:			str										=       cls.get_config_dict(lowercase_				,			**lowercase_    )

										# get the vision config dict if we are loading from AltCLIPConfig
										if config_dict.get("""model_type"""    ) == "altclip":
																	lowercase_					:			List[str]										=       config_dict["""vision_config"""]

										if "model_type" in config_dict and hasattr(cls				,			"""model_type"""    ) and config_dict["model_type"] != cls.model_type:
																	logger.warning(
																	    f'''You are using a model of type {config_dict['model_type']} to instantiate a model of type '''
																	    f'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''    )

										return cls.from_dict(lowercase_				,			**lowercase_    )





class     __magic_name__						(    _UpperCAmelCase):
			UpperCamelCase__				=       '''altclip'''
			UpperCamelCase__				=       True







			def __init__(	self				:  Optional[int]				,			lowercase_				:  Dict=None				,			lowercase_				:  List[Any]=None				,			lowercase_				:  Tuple=768				,			lowercase_				:  List[str]=2.65_92				,			**lowercase_				:  List[Any]    ):
										# If `_config_dict` exist, we use them for the backward compatibility.
										# We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
										# of confusion!).
										lowercase_					:			Dict										=       kwargs.pop("""text_config_dict"""				,			lowercase_    )
										lowercase_					:			str										=       kwargs.pop("""vision_config_dict"""				,			lowercase_    )

										super().__init__(**lowercase_    )

										# Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
										# `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
										# cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
										if text_config_dict is not None:
																	if text_config is None:
																								lowercase_					:			Dict										=       {}

																	# This is the complete result when using `text_config_dict`.
																	lowercase_					:			List[str]										=       AltCLIPTextConfig(**lowercase_    ).to_dict()

																	# Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
																	for key, value in _text_config_dict.items():
																								if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
																															# If specified in `text_config_dict`
																															if key in text_config_dict:
																																						lowercase_					:			Optional[Any]										=       (
																																						    f'''`{key}` is found in both `text_config_dict` and `text_config` but with different values. '''
																																						    f'''The value `text_config_dict["{key}"]` will be used instead.'''
																																						)
																															# If inferred from default argument values (just to be super careful)
																															else:
																																						lowercase_					:			Tuple										=       (
																																						    f'''`text_config_dict` is provided which will be used to initialize `AltCLIPTextConfig`. The '''
																																						    f'''value `text_config["{key}"]` will be overriden.'''
																																						)
																															logger.warning(lowercase_    )

            # Update all values in `text_config` with the ones in `_text_config_dict`.
																	text_config.update(_text_config_dict    )

										if vision_config_dict is not None:
																	if vision_config is None:
																								lowercase_					:			int										=       {}

																	# This is the complete result when using `vision_config_dict`.
																	lowercase_					:			List[str]										=       AltCLIPVisionConfig(**lowercase_    ).to_dict()
																	# convert keys to string instead of integer
																	if "id2label" in _vision_config_dict:
																								lowercase_					:			List[str]										=       {
																								    str(lowercase_    ): value for key, value in _vision_config_dict["""id2label"""].items()
																								}

																	# Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
																	for key, value in _vision_config_dict.items():
																								if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
																															# If specified in `vision_config_dict`
																															if key in vision_config_dict:
																																						lowercase_					:			Any										=       (
																																						    f'''`{key}` is found in both `vision_config_dict` and `vision_config` but with different '''
																																						    f'''values. The value `vision_config_dict["{key}"]` will be used instead.'''
																																						)
																															# If inferred from default argument values (just to be super careful)
																															else:
																																						lowercase_					:			List[str]										=       (
																																						    f'''`vision_config_dict` is provided which will be used to initialize `AltCLIPVisionConfig`. '''
																																						    f'''The value `vision_config["{key}"]` will be overriden.'''
																																						)
																															logger.warning(lowercase_    )

            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
																	vision_config.update(_vision_config_dict    )

										if text_config is None:
																	lowercase_					:			int										=       {}
																	logger.info("""`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values."""    )

										if vision_config is None:
																	lowercase_					:			Optional[int]										=       {}
																	logger.info("""`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values."""    )

										lowercase_					:			Optional[int]										=       AltCLIPTextConfig(**lowercase_    )
										lowercase_					:			Any										=       AltCLIPVisionConfig(**lowercase_    )

										lowercase_					:			List[Any]										=       projection_dim
										lowercase_					:			Optional[Any]										=       logit_scale_init_value
										lowercase_					:			int										=       1.0







			@classmethod
			def 	SCREAMING_SNAKE_CASE_     (	cls				:  Dict				,			lowercase_				:  AltCLIPTextConfig				,			lowercase_				:  AltCLIPVisionConfig				,			**lowercase_				:  Optional[int]    ):
										return cls(text_config=text_config.to_dict()				,			vision_config=vision_config.to_dict()				,			**lowercase_    )







			def 	SCREAMING_SNAKE_CASE_     (	self				:  Optional[int]    ):
										lowercase_					:			Union[str, Any]										=       copy.deepcopy(self.__dict__    )
										lowercase_					:			Optional[int]										=       self.text_config.to_dict()
										lowercase_					:			Any										=       self.vision_config.to_dict()
										lowercase_					:			List[str]										=       self.__class__.model_type
										return output





