





"""simple docstring"""




from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


UpperCamelCase_	 =logging.get_logger(__name__)

UpperCamelCase_	 ={"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_	 ={
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase_	 ={
    """openai/clip-vit-base-patch32""": 77,
}




class 					_a		( _lowerCAmelCase							):
				UpperCamelCase								=       VOCAB_FILES_NAMES
				UpperCamelCase								=       PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase								=       ['''input_ids''', '''attention_mask''']
				UpperCamelCase								=       CLIPTokenizer






				def __init__(					self						:       str,					lowerCAmelCase__						:       List[str]=None,					lowerCAmelCase__						:       Any=None,					lowerCAmelCase__						:       Tuple=None,					lowerCAmelCase__						:       List[str]="<|endoftext|>",					lowerCAmelCase__						:       int="<|startoftext|>",					lowerCAmelCase__						:       Union[str, Any]="<|endoftext|>",					lowerCAmelCase__						:       Optional[int]="<|endoftext|>",					**lowerCAmelCase__						:       str,					)      -> str:




										'''simple docstring'''



										super().__init__(
										    lowerCAmelCase__,					lowerCAmelCase__,					tokenizer_file=lowerCAmelCase__,					unk_token=lowerCAmelCase__,					bos_token=lowerCAmelCase__,					eos_token=lowerCAmelCase__,					pad_token=lowerCAmelCase__,					**lowerCAmelCase__,					)

										if not isinstance(self.backend_tokenizer.pre_tokenizer,					pre_tokenizers.Sequence				):
																raise ValueError(
																    '''The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'''
																    ''' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'''
																    ''' to be compatible with this version.The easiest way to do so is'''
																    ''' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'''
																    ''' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'''
																    ''' transformers.'''				)

										self._wrap_decode_method_backend_tokenizer()






				def 							snake_case      (					self						:       Dict				)      -> Tuple:




										'''simple docstring'''



										_UpperCamelCase     :   Optional[int]      =       self.backend_tokenizer.decode

										def new_decode_method(*lowerCAmelCase__						:       List[str],					**lowerCAmelCase__						:       int				):
																_UpperCamelCase     :   str      =       orig_decode_method(*lowerCAmelCase__,					**lowerCAmelCase__				)
																_UpperCamelCase     :   List[str]      =       text.replace(self.backend_tokenizer.model.end_of_word_suffix,					''' '''				).strip()
																return text

										_UpperCamelCase     :   Optional[Any]      =       new_decode_method






				def 							snake_case      (					self						:       int,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None				)      -> List[int]:




										'''simple docstring'''



										_UpperCamelCase     :   Optional[int]      =       [self.bos_token_id]
										_UpperCamelCase     :   Any      =       [self.eos_token_id]

										if token_ids_a is None:
																return bos_token + token_ids_a + eos_token
										return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token






				def 							snake_case      (					self						:       Dict,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None				)      -> List[int]:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       [self.bos_token_id]
										_UpperCamelCase     :   Optional[int]      =       [self.eos_token_id]

										if token_ids_a is None:
																return len(bos_token + token_ids_a + eos_token				) * [0]
										return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token				) * [0]






				def 							snake_case      (					self						:       List[str],					lowerCAmelCase__						:       str,					lowerCAmelCase__						:       Optional[str] = None				)      -> Tuple[str]:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       self._tokenizer.model.save(lowerCAmelCase__,					name=lowerCAmelCase__				)
										return tuple(lowerCAmelCase__				)





import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin







class 							lowercase						(   SCREAMING_SNAKE_CASE__						):
							lowercase_			:			Dict			   =['''image_processor''', '''tokenizer''']
							lowercase_			:			List[str]			   ='''AutoImageProcessor'''
							lowercase_			:			Optional[Any]			   ='''AutoTokenizer'''




							def __init__(							self  ,A__=None  ,A__=None  ,**A__):
												lowercase		      =		None
												if "feature_extractor" in kwargs:
																	warnings.warn(
																	    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
																	    ''' instead.'''  ,A__  ,)
																	lowercase		      =		kwargs.pop('''feature_extractor''')

												lowercase		      =		image_processor if image_processor is not None else feature_extractor
												if image_processor is None:
																	raise ValueError('''You need to specify an `image_processor`.''')
												if tokenizer is None:
																	raise ValueError('''You need to specify a `tokenizer`.''')

												super().__init__(A__  ,A__)
												lowercase		      =		self.image_processor
												lowercase		      =		False




							def __call__(							self  ,*A__  ,**A__):
												# For backward compatibility
												if self._in_target_context_manager:
																	return self.current_processor(*A__  ,**A__)

												lowercase		      =		kwargs.pop('''images'''  ,A__)
												lowercase		      =		kwargs.pop('''text'''  ,A__)
												if len(A__) > 0:
																	lowercase		      =		args[0]
																	lowercase		      =		args[1:]

												if images is None and text is None:
																	raise ValueError('''You need to specify either an `images` or `text` input to process.''')

												if images is not None:
																	lowercase		      =		self.image_processor(A__  ,*A__  ,**A__)
												if text is not None:
																	lowercase		      =		self.tokenizer(A__  ,**A__)

												if text is None:
																	return inputs
												elif images is None:
																	return encodings
												else:
																	lowercase		      =		encodings['''input_ids''']
																	return inputs




							def 					A__ (							self  ,*A__  ,**A__):
												return self.tokenizer.batch_decode(*A__  ,**A__)




							def 					A__ (							self  ,*A__  ,**A__):
												return self.tokenizer.decode(*A__  ,**A__)




							@contextmanager
							def 					A__ (							self):
												warnings.warn(
												    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
												    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
												    '''your images inputs, or in a separate call.''')
												lowercase		      =		True
												lowercase		      =		self.tokenizer
												yield
												lowercase		      =		self.image_processor
												lowercase		      =		False




							@property
							def 					A__ (							self):
												warnings.warn(
												    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''  ,A__  ,)
												return self.image_processor_class





							@property
							def 					A__ (							self):
												warnings.warn(
												    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''  ,A__  ,)
												return self.image_processor




