






"""simple docstring"""



from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


_UpperCamelCase						:    Optional[int]		=						logging.get_logger(__name__)

_UpperCamelCase						:    str		=						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase						:    Union[str, Any]		=						{
    'vocab_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json',
    },
    'merges_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'openai/clip-vit-base-patch32': (
            'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'
        ),
    },
}

_UpperCamelCase						:    Union[str, Any]		=						{
    'openai/clip-vit-base-patch32': 7_7,
}

class 			a	(  a_      ):
    UpperCAmelCase_					:	Tuple      	=VOCAB_FILES_NAMES
    UpperCAmelCase_					:	Dict      	=PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase_					:	int      	=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase_					:	Union[str, Any]      	=["input_ids", "attention_mask"]
    UpperCAmelCase_					:	int      	=CLIPTokenizer







    def __init__(     self				,						_lowerCamelCase=None				,						_lowerCamelCase=None				,						_lowerCamelCase=None				,						_lowerCamelCase="<|endoftext|>"				,						_lowerCamelCase="<|startoftext|>"				,						_lowerCamelCase="<|endoftext|>"				,						_lowerCamelCase="<|endoftext|>"				,						**_lowerCamelCase				,						):
      super().__init__(
          _lowerCamelCase				,						_lowerCamelCase				,						tokenizer_file=_lowerCamelCase				,						unk_token=_lowerCamelCase				,						bos_token=_lowerCamelCase				,						eos_token=_lowerCamelCase				,						pad_token=_lowerCamelCase				,						**_lowerCamelCase				,						)

      if not isinstance(self.backend_tokenizer.pre_tokenizer				,						pre_tokenizers.Sequence			):
        raise ValueError(
            'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
            ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
            ' to be compatible with this version.The easiest way to do so is'
            ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
            ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
            ' transformers.'			)

      self._wrap_decode_method_backend_tokenizer()







    def      UpperCamelCase_ (     self			):
      lowercase				       =			self.backend_tokenizer.decode

      def new_decode_method(*_lowerCamelCase				,						**_lowerCamelCase			):
        lowercase				       =			orig_decode_method(*_lowerCamelCase				,						**_lowerCamelCase			)
        lowercase				       =			text.replace(self.backend_tokenizer.model.end_of_word_suffix				,						' '			).strip()
        return text

      lowercase				       =			new_decode_method







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      lowercase				       =			[self.bos_token_id]
      lowercase				       =			[self.eos_token_id]

      if token_ids_a is None:
        return bos_token + token_ids_a + eos_token
      return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      lowercase				       =			[self.bos_token_id]
      lowercase				       =			[self.eos_token_id]

      if token_ids_a is None:
        return len(bos_token + token_ids_a + eos_token			) * [0]
      return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token			) * [0]







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      lowercase				       =			self._tokenizer.model.save(_lowerCamelCase				,						name=_lowerCamelCase			)
      return tuple(_lowerCamelCase			)




"""simple docstring"""
from transformers import ProcessorMixin




class  _snake_case	(    _lowercase							):
							SCREAMING_SNAKE_CASE		  =    '''AutoFeatureExtractor'''
							SCREAMING_SNAKE_CASE		  =    '''AutoTokenizer'''
