






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowerCamelCase	:							List[str]				=logging.get_logger(__name__)

lowerCamelCase	:							int				={'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase	:							Optional[int]				={
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase	:							List[Any]				={
    '''Salesforce/codegen-350M-mono''': 2048,
}





class 			__a  (    A__       ):
 _lowerCAmelCase       :       Tuple        = VOCAB_FILES_NAMES
 _lowerCAmelCase       :       List[Any]        = PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase       :       List[str]        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase       :       List[str]        = ['''input_ids''', '''attention_mask''']
 _lowerCAmelCase       :       Any        = CodeGenTokenizer







 def __init__(		self					:   Union[str, Any]							,   SCREAMING_SNAKE_CASE					:   List[Any]=None							,   SCREAMING_SNAKE_CASE					:   List[Any]=None							,   SCREAMING_SNAKE_CASE					:   Optional[Any]=None							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]="<|endoftext|>"							,   SCREAMING_SNAKE_CASE					:   List[Any]="<|endoftext|>"							,   SCREAMING_SNAKE_CASE					:   Optional[int]="<|endoftext|>"							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=False							,   **SCREAMING_SNAKE_CASE					:   List[Any]							,   ):


      '''simple docstring'''






      super().__init__(
          SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   tokenizer_file=SCREAMING_SNAKE_CASE							,   unk_token=SCREAMING_SNAKE_CASE							,   bos_token=SCREAMING_SNAKE_CASE							,   eos_token=SCREAMING_SNAKE_CASE							,   add_prefix_space=SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE							,   )

      if kwargs.pop("add_bos_token"							,   SCREAMING_SNAKE_CASE						):
           UpperCamelCase__			:   List[Any]       				=  kwargs.pop("name_or_path"							,   ""						)
           raise ValueError(
               "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
               "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
               F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
               F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
               "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
               " so that the fast tokenizer works correctly."						)

      UpperCamelCase__			:   List[Any]       				=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
      if pre_tok_state.get("add_prefix_space"							,   SCREAMING_SNAKE_CASE						) != add_prefix_space:
           UpperCamelCase__			:   int       				=  getattr(SCREAMING_SNAKE_CASE							,   pre_tok_state.pop("type"						)						)
           UpperCamelCase__			:   str       				=  add_prefix_space
           UpperCamelCase__			:   List[str]       				=  pre_tok_class(**SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Union[str, Any]       				=  add_prefix_space







 def 					__lowercase					(		self					:   List[str]							,   *SCREAMING_SNAKE_CASE					:   str							,   **SCREAMING_SNAKE_CASE					:   Dict						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  kwargs.get("is_split_into_words"							,   SCREAMING_SNAKE_CASE						)
      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   List[str]							,   *SCREAMING_SNAKE_CASE					:   Any							,   **SCREAMING_SNAKE_CASE					:   Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[int]       				=  kwargs.get("is_split_into_words"							,   SCREAMING_SNAKE_CASE						)

      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   str							,   SCREAMING_SNAKE_CASE					:   Optional[str] = None						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  self._tokenizer.model.save(SCREAMING_SNAKE_CASE							,   name=SCREAMING_SNAKE_CASE						)
      return tuple(SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Tuple							,   SCREAMING_SNAKE_CASE					:   Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]							,   SCREAMING_SNAKE_CASE					:   bool = False							,   SCREAMING_SNAKE_CASE					:   bool = None							,   SCREAMING_SNAKE_CASE					:   Optional[List[str]] = None							,   **SCREAMING_SNAKE_CASE					:   List[Any]							,   ):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  super().decode(
          token_ids=SCREAMING_SNAKE_CASE							,   skip_special_tokens=SCREAMING_SNAKE_CASE							,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE							,   )

      if truncate_before_pattern is not None and len(SCREAMING_SNAKE_CASE						) > 0:
           UpperCamelCase__			:   Tuple       				=  self.truncate(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)

      return decoded_text







 def 					__lowercase					(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   int						):


      '''simple docstring'''






      def find_re(SCREAMING_SNAKE_CASE					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   List[str]							,   SCREAMING_SNAKE_CASE					:   str						):
           UpperCamelCase__			:   Optional[int]       				=  pattern.search(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
           return m.start() if m else -1

      UpperCamelCase__			:   int       				=  [re.compile(SCREAMING_SNAKE_CASE							,   re.MULTILINE						) for pattern in truncate_before_pattern]

      UpperCamelCase__			:   Any       				=  list(re.finditer("^print"							,   SCREAMING_SNAKE_CASE							,   re.MULTILINE						)						)

      if len(SCREAMING_SNAKE_CASE						) > 1:
           UpperCamelCase__			:   Tuple       				=  completion[: prints[1].start()]

      UpperCamelCase__			:   int       				=  list(re.finditer("^def"							,   SCREAMING_SNAKE_CASE							,   re.MULTILINE						)						)

      if len(SCREAMING_SNAKE_CASE						) > 1:
           UpperCamelCase__			:   List[str]       				=  completion[: defs[1].start()]

      UpperCamelCase__			:   Dict       				=  0

      UpperCamelCase__			:   List[Any]       				=  [
          pos for pos in [find_re(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						) for terminal in terminals] if pos != -1
      ]

      if len(SCREAMING_SNAKE_CASE						) > 0:
           return completion[: min(SCREAMING_SNAKE_CASE						)]
      else:
           return completion







import copy
import os
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Union


if TYPE_CHECKING:
 from ...processing_utils import ProcessorMixin
 from ...utils import TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase	:							Optional[Any]				=logging.get_logger(__name__)

lowerCamelCase	:							Optional[int]				={
    '''google/owlvit-base-patch32''': '''https://huggingface.co/google/owlvit-base-patch32/resolve/main/config.json''',
    '''google/owlvit-base-patch16''': '''https://huggingface.co/google/owlvit-base-patch16/resolve/main/config.json''',
    '''google/owlvit-large-patch14''': '''https://huggingface.co/google/owlvit-large-patch14/resolve/main/config.json''',
}





class 			__a  (    A__       ):
 _lowerCAmelCase       :       Optional[int]        = '''owlvit_text_model'''







 def __init__(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=4_94_08							,   SCREAMING_SNAKE_CASE					:   List[str]=5_12							,   SCREAMING_SNAKE_CASE					:   List[Any]=20_48							,   SCREAMING_SNAKE_CASE					:   Any=12							,   SCREAMING_SNAKE_CASE					:   Any=8							,   SCREAMING_SNAKE_CASE					:   Dict=16							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]="quick_gelu"							,   SCREAMING_SNAKE_CASE					:   List[str]=1e-5							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=0.0							,   SCREAMING_SNAKE_CASE					:   Any=0.0_2							,   SCREAMING_SNAKE_CASE					:   int=1.0							,   SCREAMING_SNAKE_CASE					:   Any=0							,   SCREAMING_SNAKE_CASE					:   int=4_94_06							,   SCREAMING_SNAKE_CASE					:   List[str]=4_94_07							,   **SCREAMING_SNAKE_CASE					:   Optional[Any]							,   ):


      '''simple docstring'''






      super().__init__(pad_token_id=SCREAMING_SNAKE_CASE							,   bos_token_id=SCREAMING_SNAKE_CASE							,   eos_token_id=SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Tuple       				=  vocab_size
      UpperCamelCase__			:   int       				=  hidden_size
      UpperCamelCase__			:   List[str]       				=  intermediate_size
      UpperCamelCase__			:   Tuple       				=  num_hidden_layers
      UpperCamelCase__			:   str       				=  num_attention_heads
      UpperCamelCase__			:   Any       				=  max_position_embeddings
      UpperCamelCase__			:   List[Any]       				=  hidden_act
      UpperCamelCase__			:   str       				=  layer_norm_eps
      UpperCamelCase__			:   List[Any]       				=  attention_dropout
      UpperCamelCase__			:   Tuple       				=  initializer_range
      UpperCamelCase__			:   Optional[Any]       				=  initializer_factor







 @classmethod
 def 					__lowercase					(		cls					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   Union[str, os.PathLike]							,   **SCREAMING_SNAKE_CASE					:   Optional[Any]						):


      '''simple docstring'''






      cls._set_token_in_kwargs(SCREAMING_SNAKE_CASE						)

      UpperCamelCase__				,     UpperCamelCase__			:   Any       				=  cls.get_config_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)

      # get the text config dict if we are loading from OwlViTConfig
      if config_dict.get("model_type"						) == "owlvit":
           UpperCamelCase__			:   Dict       				=  config_dict["text_config"]

      if "model_type" in config_dict and hasattr(cls							,   "model_type"						) and config_dict["model_type"] != cls.model_type:
           logger.warning(
               F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
               F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'						)

      return cls.from_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







class 			__a  (    A__       ):
 _lowerCAmelCase       :       str        = '''owlvit_vision_model'''







 def __init__(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   str=7_68							,   SCREAMING_SNAKE_CASE					:   Dict=30_72							,   SCREAMING_SNAKE_CASE					:   int=12							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=12							,   SCREAMING_SNAKE_CASE					:   Dict=3							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=7_68							,   SCREAMING_SNAKE_CASE					:   Optional[int]=32							,   SCREAMING_SNAKE_CASE					:   Dict="quick_gelu"							,   SCREAMING_SNAKE_CASE					:   Optional[Any]=1e-5							,   SCREAMING_SNAKE_CASE					:   List[str]=0.0							,   SCREAMING_SNAKE_CASE					:   Dict=0.0_2							,   SCREAMING_SNAKE_CASE					:   Optional[int]=1.0							,   **SCREAMING_SNAKE_CASE					:   Tuple							,   ):


      '''simple docstring'''






      super().__init__(**SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Any       				=  hidden_size
      UpperCamelCase__			:   str       				=  intermediate_size
      UpperCamelCase__			:   Any       				=  num_hidden_layers
      UpperCamelCase__			:   str       				=  num_attention_heads
      UpperCamelCase__			:   int       				=  num_channels
      UpperCamelCase__			:   Union[str, Any]       				=  image_size
      UpperCamelCase__			:   List[Any]       				=  patch_size
      UpperCamelCase__			:   Tuple       				=  hidden_act
      UpperCamelCase__			:   Optional[int]       				=  layer_norm_eps
      UpperCamelCase__			:   Optional[Any]       				=  attention_dropout
      UpperCamelCase__			:   Dict       				=  initializer_range
      UpperCamelCase__			:   int       				=  initializer_factor







 @classmethod
 def 					__lowercase					(		cls					:   Dict							,   SCREAMING_SNAKE_CASE					:   Union[str, os.PathLike]							,   **SCREAMING_SNAKE_CASE					:   List[Any]						):


      '''simple docstring'''






      cls._set_token_in_kwargs(SCREAMING_SNAKE_CASE						)

      UpperCamelCase__				,     UpperCamelCase__			:   List[Any]       				=  cls.get_config_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)

      # get the vision config dict if we are loading from OwlViTConfig
      if config_dict.get("model_type"						) == "owlvit":
           UpperCamelCase__			:   List[str]       				=  config_dict["vision_config"]

      if "model_type" in config_dict and hasattr(cls							,   "model_type"						) and config_dict["model_type"] != cls.model_type:
           logger.warning(
               F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
               F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'						)

      return cls.from_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







class 			__a  (    A__       ):
 _lowerCAmelCase       :       str        = '''owlvit'''
 _lowerCAmelCase       :       Tuple        = True







 def __init__(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   Optional[Any]=None							,   SCREAMING_SNAKE_CASE					:   int=None							,   SCREAMING_SNAKE_CASE					:   str=5_12							,   SCREAMING_SNAKE_CASE					:   Any=2.6_5_9_2							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=True							,   **SCREAMING_SNAKE_CASE					:   List[Any]							,   ):


      '''simple docstring'''






      super().__init__(**SCREAMING_SNAKE_CASE						)

      if text_config is None:
           UpperCamelCase__			:   str       				=  {}
           logger.info("text_config is None. Initializing the OwlViTTextConfig with default values."						)

      if vision_config is None:
           UpperCamelCase__			:   List[str]       				=  {}
           logger.info("vision_config is None. initializing the OwlViTVisionConfig with default values."						)

      UpperCamelCase__			:   Dict       				=  OwlViTTextConfig(**SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Dict       				=  OwlViTVisionConfig(**SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Dict       				=  projection_dim
      UpperCamelCase__			:   Union[str, Any]       				=  logit_scale_init_value
      UpperCamelCase__			:   int       				=  return_dict
      UpperCamelCase__			:   Tuple       				=  1.0







 @classmethod
 def 					__lowercase					(		cls					:   Any							,   SCREAMING_SNAKE_CASE					:   Union[str, os.PathLike]							,   **SCREAMING_SNAKE_CASE					:   Dict						):


      '''simple docstring'''






      cls._set_token_in_kwargs(SCREAMING_SNAKE_CASE						)

      UpperCamelCase__				,     UpperCamelCase__			:   Dict       				=  cls.get_config_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)

      if "model_type" in config_dict and hasattr(cls							,   "model_type"						) and config_dict["model_type"] != cls.model_type:
           logger.warning(
               F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
               F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'						)

      return cls.from_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







 @classmethod
 def 					__lowercase					(		cls					:   int							,   SCREAMING_SNAKE_CASE					:   Dict							,   SCREAMING_SNAKE_CASE					:   Dict							,   **SCREAMING_SNAKE_CASE					:   Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  {}
      UpperCamelCase__			:   Union[str, Any]       				=  text_config
      UpperCamelCase__			:   Optional[int]       				=  vision_config

      return cls.from_dict(SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[int]       				=  copy.deepcopy(self.__dict__						)
      UpperCamelCase__			:   Union[str, Any]       				=  self.text_config.to_dict()
      UpperCamelCase__			:   List[str]       				=  self.vision_config.to_dict()
      UpperCamelCase__			:   Optional[int]       				=  self.__class__.model_type
      return output







class 			__a  (    A__       ):







 @property
 def 					__lowercase					(		self					:   Any						):


      '''simple docstring'''






      return OrderedDict(
          [
              ("input_ids", {0: "batch", 1: "sequence"}),
              ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
              ("attention_mask", {0: "batch", 1: "sequence"}),
          ]						)







 @property
 def 					__lowercase					(		self					:   Optional[int]						):


      '''simple docstring'''






      return OrderedDict(
          [
              ("logits_per_image", {0: "batch"}),
              ("logits_per_text", {0: "batch"}),
              ("text_embeds", {0: "batch"}),
              ("image_embeds", {0: "batch"}),
          ]						)







 @property
 def 					__lowercase					(		self					:   Dict						):


      '''simple docstring'''






      return 1e-4







 def 					__lowercase					(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   "ProcessorMixin"							,   SCREAMING_SNAKE_CASE					:   int = -1							,   SCREAMING_SNAKE_CASE					:   int = -1							,   SCREAMING_SNAKE_CASE					:   Optional["TensorType"] = None							,   ):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[int]       				=  super().generate_dummy_inputs(
          processor.tokenizer							,   batch_size=SCREAMING_SNAKE_CASE							,   seq_length=SCREAMING_SNAKE_CASE							,   framework=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   List[str]       				=  super().generate_dummy_inputs(
          processor.image_processor							,   batch_size=SCREAMING_SNAKE_CASE							,   framework=SCREAMING_SNAKE_CASE						)
      return {**text_input_dict, **image_input_dict}







 @property
 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      return 14