






from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


A_			:Optional[int]	    =		logging.get_logger(__name__)




class 							__A	(     nn.Module					):





				"""simple docstring"""







				UpperCamelCase__					: Union[str, Any]     					=4_2
				UpperCamelCase__					: Dict     					=jnp.floataa


				def  __lowercase				(					self       ):




									"""simple docstring"""

									__UpperCamelCase :   List[Any]									=self.config.text_config
									__UpperCamelCase :   Any									=self.config.vision_config

									__UpperCamelCase :   Optional[Any]									=self.config.projection_dim
									__UpperCamelCase :   str									=text_config.hidden_size
									__UpperCamelCase :   str									=vision_config.hidden_size

									__UpperCamelCase :   List[Any]									=FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
									__UpperCamelCase :   List[str]									=FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__				,    a__       ).module_class

									__UpperCamelCase :   Tuple									=text_module(a__				,    dtype=self.dtype       )
									__UpperCamelCase :   int									=vision_module(a__				,    dtype=self.dtype       )

									__UpperCamelCase :   Optional[int]									=nn.Dense(
									    self.projection_dim				,    dtype=self.dtype				,    kernel_init=jax.nn.initializers.normal(0.02       )				,    use_bias=a__				,    )
									__UpperCamelCase :   List[Any]									=nn.Dense(
									    self.projection_dim				,    dtype=self.dtype				,    kernel_init=jax.nn.initializers.normal(0.02       )				,    use_bias=a__				,    )
									__UpperCamelCase :   Optional[Any]									=self.param('logit_scale'				,    jax.nn.initializers.ones				,    []       )







				def __call__(					self				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__ = True				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    ):




									"""simple docstring"""

									__UpperCamelCase :   Optional[Any]									=return_dict if return_dict is not None else self.config.return_dict

									__UpperCamelCase :   List[Any]									=self.vision_model(
									    pixel_values=a__				,    deterministic=a__				,    output_attentions=a__				,    output_hidden_states=a__				,    return_dict=a__				,    )

									__UpperCamelCase :   Any									=self.text_model(
									    input_ids=a__				,    attention_mask=a__				,    token_type_ids=a__				,    position_ids=a__				,    deterministic=a__				,    output_attentions=a__				,    output_hidden_states=a__				,    return_dict=a__				,    )

									__UpperCamelCase :   List[Any]									=vision_outputs[1]
									__UpperCamelCase :   int									=self.visual_projection(a__       )

									__UpperCamelCase :   List[Any]									=text_outputs[1]
									__UpperCamelCase :   Dict									=self.text_projection(a__       )

									# normalized features
									__UpperCamelCase :   Tuple									=image_embeds / jnp.linalg.norm(a__				,    axis=-1				,    keepdims=a__       )
									__UpperCamelCase :   str									=text_embeds / jnp.linalg.norm(a__				,    axis=-1				,    keepdims=a__       )

									# cosine similarity as logits
									__UpperCamelCase :   List[Any]									=jnp.exp(self.logit_scale       )
									__UpperCamelCase :   Optional[int]									=jnp.matmul(a__				,    image_embeds.T       ) * logit_scale
									__UpperCamelCase :   Optional[Any]									=logits_per_text.T

									if not return_dict:
														return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

									return FlaxCLIPOutput(
									    logits_per_image=a__				,    logits_per_text=a__				,    text_embeds=a__				,    image_embeds=a__				,    text_model_output=a__				,    vision_model_output=a__				,    )




class 							__A	(     _UpperCamelCase					):





				"""simple docstring"""







				UpperCamelCase__					: Optional[int]     					=HybridCLIPConfig
				UpperCamelCase__					: str     					=FlaxHybridCLIPModule


				def __init__(					self				,    lowerCamelCase__				,    lowerCamelCase__ = None				,    lowerCamelCase__ = 0				,    lowerCamelCase__ = jnp.floataa				,    **lowerCamelCase__				,    ):




									"""simple docstring"""

									if input_shape is None:
														__UpperCamelCase :   str									=((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

									__UpperCamelCase :   Union[str, Any]									=self.module_class(config=a__				,    dtype=a__				,    **a__       )
									super().__init__(a__				,    a__				,    input_shape=a__				,    seed=a__				,    dtype=a__       )


				def  __lowercase				(					self				,    lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__ = None       ):




									"""simple docstring"""

									__UpperCamelCase :   List[str]									=jnp.zeros(input_shape[0]				,    dtype='i4'       )
									__UpperCamelCase :   Optional[Any]									=jnp.broadcast_to(jnp.arange(jnp.atleast_ad(a__       ).shape[-1]       )				,    input_shape[0]       )
									__UpperCamelCase :   Any									=jnp.ones_like(a__       )
									__UpperCamelCase :   Dict									=jnp.ones_like(a__       )

									__UpperCamelCase :   Union[str, Any]									=jax.random.normal(a__				,    input_shape[1]       )

									__UpperCamelCase						,							__UpperCamelCase :   Optional[Any]									=jax.random.split(a__       )
									__UpperCamelCase :   List[str]									={'params': params_rng, 'dropout': dropout_rng}

									return self.module.init(a__				,    a__				,    a__				,    a__				,    a__				,    a__       )["params"]


				def __call__(					self				,    lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__ = None				,    lowerCamelCase__ = None				,    lowerCamelCase__ = False				,    lowerCamelCase__ = None				,    lowerCamelCase__ = None				,    lowerCamelCase__ = None				,    ):




									"""simple docstring"""

									__UpperCamelCase :   List[str]									=output_attentions if output_attentions is not None else self.config.output_attentions
									__UpperCamelCase :   str									=(
									    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
									)
									__UpperCamelCase :   List[Any]									=return_dict if return_dict is not None else self.config.return_dict

									if position_ids is None:
														__UpperCamelCase :   Dict									=jnp.broadcast_to(jnp.arange(jnp.atleast_ad(a__       ).shape[-1]       )				,    input_ids.shape       )

									if token_type_ids is None:
														__UpperCamelCase :   Tuple									=jnp.zeros_like(a__       )

									if attention_mask is None:
														__UpperCamelCase :   Optional[Any]									=jnp.ones_like(a__       )

									# Handle any PRNG if needed
									__UpperCamelCase :   Union[str, Any]									={}
									if dropout_rng is not None:
														__UpperCamelCase :   Optional[int]									=dropout_rng

									return self.module.apply(
									    {'params': params or self.params}				,    jnp.array(a__				,    dtype='i4'       )				,    jnp.array(a__				,    dtype=jnp.floataa       )				,    jnp.array(a__				,    dtype='i4'       )				,    jnp.array(a__				,    dtype='i4'       )				,    jnp.array(a__				,    dtype='i4'       )				,    not train				,    a__				,    a__				,    a__				,    rngs=a__				,    )


				def  __lowercase				(					self				,    lowerCamelCase__				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__=None				,    lowerCamelCase__ = None				,    lowerCamelCase__ = None				,    lowerCamelCase__=False				,    ):




									"""simple docstring"""

									if position_ids is None:
														__UpperCamelCase :   Optional[int]									=jnp.broadcast_to(jnp.arange(jnp.atleast_ad(a__       ).shape[-1]       )				,    input_ids.shape       )

									if token_type_ids is None:
														__UpperCamelCase :   Union[str, Any]									=jnp.zeros_like(a__       )

									if attention_mask is None:
														__UpperCamelCase :   Optional[Any]									=jnp.ones_like(a__       )

									# Handle any PRNG if needed
									__UpperCamelCase :   Dict									={}
									if dropout_rng is not None:
														__UpperCamelCase :   int									=dropout_rng

									def _get_features(lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__       ):
														__UpperCamelCase :   Union[str, Any]									=module.text_model(
														    input_ids=a__				,    attention_mask=a__				,    position_ids=a__				,    token_type_ids=a__				,    deterministic=a__				,    )
														__UpperCamelCase :   List[str]									=text_outputs[1]
														__UpperCamelCase :   Dict									=module.text_projection(a__       )
														return text_features

									return self.module.apply(
									    {'params': params or self.params}				,    jnp.array(a__				,    dtype='i4'       )				,    jnp.array(a__				,    dtype='i4'       )				,    jnp.array(a__				,    dtype='i4'       )				,    jnp.array(a__				,    dtype='i4'       )				,    not train				,    method=_get_features				,    rngs=a__				,    )


				def  __lowercase				(					self				,    lowerCamelCase__				,    lowerCamelCase__ = None				,    lowerCamelCase__ = None				,    lowerCamelCase__=False       ):




									"""simple docstring"""

									__UpperCamelCase :   List[str]									={}
									if dropout_rng is not None:
														__UpperCamelCase :   Any									=dropout_rng

									def _get_features(lowerCamelCase__				,    lowerCamelCase__				,    lowerCamelCase__       ):
														__UpperCamelCase :   Optional[Any]									=module.vision_model(pixel_values=a__				,    deterministic=a__       )
														__UpperCamelCase :   Optional[Any]									=vision_outputs[1]  # pooled_output
														__UpperCamelCase :   Tuple									=module.visual_projection(a__       )
														return image_features

									return self.module.apply(
									    {'params': params or self.params}				,    jnp.array(a__				,    dtype=jnp.floataa       )				,    not train				,    method=_get_features				,    rngs=a__				,    )







				@classmethod
				def  __lowercase				(					cls				,    lowerCamelCase__ = None				,    lowerCamelCase__ = None				,    *lowerCamelCase__				,    **lowerCamelCase__				,    ):




									"""simple docstring"""

									__UpperCamelCase :   List[Any]									={
									    argument[len('text_'       ) :]: value for argument, value in kwargs.items() if argument.startswith('text_'       )
									}

									__UpperCamelCase :   List[Any]									={
									    argument[len('vision_'       ) :]: value for argument, value in kwargs.items() if argument.startswith('vision_'       )
									}

									# remove text, vision kwargs from kwargs
									for key in kwargs_text.keys():
														del kwargs["text_" + key]
									for key in kwargs_vision.keys():
														del kwargs["vision_" + key]

									# Load and initialize the text and vision model
									__UpperCamelCase :   int									=kwargs_text.pop('model'				,    a__       )
									if text_model is None:
														assert (
														    text_model_name_or_path is not None
														), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
														from transformers import FlaxAutoModel

														if "config" not in kwargs_text:
																			from transformers import AutoConfig

																			__UpperCamelCase :   List[Any]									=AutoConfig.from_pretrained(a__       )
																			__UpperCamelCase :   List[Any]									=text_config

														__UpperCamelCase :   Optional[int]									=FlaxAutoModel.from_pretrained(a__				,    *a__				,    **a__       )

									__UpperCamelCase :   Optional[int]									=kwargs_vision.pop('model'				,    a__       )
									if vision_model is None:
														assert (
														    vision_model_name_or_path is not None
														), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
														from transformers import FlaxAutoModel

														if "config" not in kwargs_vision:
																			from transformers import AutoConfig

																			__UpperCamelCase :   List[Any]									=AutoConfig.from_pretrained(a__       )
																			__UpperCamelCase :   Optional[int]									=vision_config

														__UpperCamelCase :   Dict									=FlaxAutoModel.from_pretrained(a__				,    *a__				,    **a__       )

									# instantiate config with corresponding kwargs
									__UpperCamelCase :   Tuple									=kwargs.pop('dtype'				,    jnp.floataa       )
									__UpperCamelCase :   Tuple									=HybridCLIPConfig.from_text_vision_configs(text_model.config				,    vision_model.config				,    **a__       )

									# init model
									__UpperCamelCase :   List[Any]									=cls(a__				,    *a__				,    dtype=a__				,    **a__       )

									if vision_config.model_type == "clip":
														__UpperCamelCase :   Optional[int]									=vision_model.params['vision_model']
														__UpperCamelCase :   Union[str, Any]									=vision_model.params['visual_projection']['kernel']
									else:
														__UpperCamelCase :   Any									=vision_model.params

									__UpperCamelCase :   Optional[Any]									=text_model.params

									return model





"""simple docstring"""


def   a__			(					__lowercase  ,      __lowercase )						->						Tuple:
				_A       	=      1
				while b > 0:
								if b & 1:
												res *= a

								a *= a
								b >>= 1

				return res






def   a__			(					__lowercase  ,      __lowercase  ,      __lowercase )						->						Optional[Any]:
				_A       	=      1
				while b > 0:
								if b & 1:
												_A       	=      ((res % c) * (a % c)) % c

								a *= a
								b >>= 1

				return res