






"""simple docstring"""


from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


lowercase_								=	logging.get_logger(__name__)


class    snake_case						(				nn.Module		):


   '''simple docstring'''

   A_		:      HybridCLIPConfig
   A_		:      jnp.dtype							=					jnp.floataa


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     __A		     =					self.config.text_config
     __A		     =					self.config.vision_config

     __A		     =					self.config.projection_dim
     __A		     =					text_config.hidden_size
     __A		     =					vision_config.hidden_size

     __A		     =					FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
     __A		     =					FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__,			_lowerCamelCase    ).module_class

     __A		     =					text_module(_lowerCamelCase,			dtype=self.dtype    )
     __A		     =					vision_module(_lowerCamelCase,			dtype=self.dtype    )

     __A		     =					nn.Dense(
         self.projection_dim,			dtype=self.dtype,			kernel_init=jax.nn.initializers.normal(0.02    ),			use_bias=_lowerCamelCase,			)
     __A		     =					nn.Dense(
         self.projection_dim,			dtype=self.dtype,			kernel_init=jax.nn.initializers.normal(0.02    ),			use_bias=_lowerCamelCase,			)
     __A		     =					self.param('''logit_scale''',			jax.nn.initializers.ones,			[]    )






   def __call__(      self					:	Any,			_lowerCamelCase					:	Union[str, Any]=None,			_lowerCamelCase					:	Union[str, Any]=None,			_lowerCamelCase					:	Optional[int]=None,			_lowerCamelCase					:	Any=None,			_lowerCamelCase					:	str=None,			_lowerCamelCase					:	bool = True,			_lowerCamelCase					:	List[str]=None,			_lowerCamelCase					:	List[Any]=None,			_lowerCamelCase					:	Tuple=None,			):

     '''simple docstring'''
     __A		     =					return_dict if return_dict is not None else self.config.return_dict

     __A		     =					self.vision_model(
         pixel_values=_lowerCamelCase,			deterministic=_lowerCamelCase,			output_attentions=_lowerCamelCase,			output_hidden_states=_lowerCamelCase,			return_dict=_lowerCamelCase,			)

     __A		     =					self.text_model(
         input_ids=_lowerCamelCase,			attention_mask=_lowerCamelCase,			token_type_ids=_lowerCamelCase,			position_ids=_lowerCamelCase,			deterministic=_lowerCamelCase,			output_attentions=_lowerCamelCase,			output_hidden_states=_lowerCamelCase,			return_dict=_lowerCamelCase,			)

     __A		     =					vision_outputs[1]
     __A		     =					self.visual_projection(_lowerCamelCase    )

     __A		     =					text_outputs[1]
     __A		     =					self.text_projection(_lowerCamelCase    )

     # normalized features
     __A		     =					image_embeds / jnp.linalg.norm(_lowerCamelCase,			axis=-1,			keepdims=_lowerCamelCase    )
     __A		     =					text_embeds / jnp.linalg.norm(_lowerCamelCase,			axis=-1,			keepdims=_lowerCamelCase    )

     # cosine similarity as logits
     __A		     =					jnp.exp(self.logit_scale    )
     __A		     =					jnp.matmul(_lowerCamelCase,			image_embeds.T    ) * logit_scale
     __A		     =					logits_per_text.T

     if not return_dict:
       return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

     return FlaxCLIPOutput(
         logits_per_image=_lowerCamelCase,			logits_per_text=_lowerCamelCase,			text_embeds=_lowerCamelCase,			image_embeds=_lowerCamelCase,			text_model_output=_lowerCamelCase,			vision_model_output=_lowerCamelCase,			)





class    snake_case						(				_lowerCAmelCase		):


   '''simple docstring'''

   A_		:      List[str]							=					HybridCLIPConfig
   A_		:      Tuple							=					FlaxHybridCLIPModule


   def __init__(      self					:	Tuple,			_lowerCamelCase					:	HybridCLIPConfig,			_lowerCamelCase					:	Optional[Tuple] = None,			_lowerCamelCase					:	int = 0,			_lowerCamelCase					:	jnp.dtype = jnp.floataa,			**_lowerCamelCase					:	Dict,			):

     '''simple docstring'''
     if input_shape is None:
       __A		     =					((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

     __A		     =					self.module_class(config=_lowerCamelCase,			dtype=_lowerCamelCase,			**_lowerCamelCase    )
     super().__init__(_lowerCamelCase,			_lowerCamelCase,			input_shape=_lowerCamelCase,			seed=_lowerCamelCase,			dtype=_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any],			_lowerCamelCase					:	jax.random.PRNGKey,			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	FrozenDict = None    ):

     '''simple docstring'''
     # init input tensor
     __A		     =					jnp.zeros(input_shape[0],			dtype='''i4'''    )
     __A		     =					jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_lowerCamelCase    ).shape[-1]    ),			input_shape[0]    )
     __A		     =					jnp.ones_like(_lowerCamelCase    )
     __A		     =					jnp.ones_like(_lowerCamelCase    )

     __A		     =					jax.random.normal(_lowerCamelCase,			input_shape[1]    )

     __A     ,       __A		     =					jax.random.split(_lowerCamelCase    )
     __A		     =					{'''params''': params_rng, '''dropout''': dropout_rng}

     return self.module.init(_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase    )["params"]


   def __call__(      self					:	Tuple,			_lowerCamelCase					:	str,			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	Any=None,			_lowerCamelCase					:	Dict=None,			_lowerCamelCase					:	List[str]=None,			_lowerCamelCase					:	dict = None,			_lowerCamelCase					:	jax.random.PRNGKey = None,			_lowerCamelCase					:	bool = False,			_lowerCamelCase					:	Optional[bool] = None,			_lowerCamelCase					:	Optional[bool] = None,			_lowerCamelCase					:	Optional[bool] = None,			):

     '''simple docstring'''
     __A		     =					output_attentions if output_attentions is not None else self.config.output_attentions
     __A		     =					(
         output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
     )
     __A		     =					return_dict if return_dict is not None else self.config.return_dict

     if position_ids is None:
       __A		     =					jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_lowerCamelCase    ).shape[-1]    ),			input_ids.shape    )

     if token_type_ids is None:
       __A		     =					jnp.zeros_like(_lowerCamelCase    )

     if attention_mask is None:
       __A		     =					jnp.ones_like(_lowerCamelCase    )

     # Handle any PRNG if needed
     __A		     =					{}
     if dropout_rng is not None:
       __A		     =					dropout_rng

     return self.module.apply(
         {'''params''': params or self.params},			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			jnp.array(_lowerCamelCase,			dtype=jnp.floataa    ),			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			not train,			_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase,			rngs=_lowerCamelCase,			)


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple,			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Optional[Any]=None,			_lowerCamelCase					:	List[str]=None,			_lowerCamelCase					:	Optional[int]=None,			_lowerCamelCase					:	dict = None,			_lowerCamelCase					:	jax.random.PRNGKey = None,			_lowerCamelCase					:	List[str]=False,			):

     '''simple docstring'''
     if position_ids is None:
       __A		     =					jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_lowerCamelCase    ).shape[-1]    ),			input_ids.shape    )

     if token_type_ids is None:
       __A		     =					jnp.zeros_like(_lowerCamelCase    )

     if attention_mask is None:
       __A		     =					jnp.ones_like(_lowerCamelCase    )

     # Handle any PRNG if needed
     __A		     =					{}
     if dropout_rng is not None:
       __A		     =					dropout_rng

     def _get_features(_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	int,			_lowerCamelCase					:	str,			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Tuple    ):
       __A		     =					module.text_model(
           input_ids=_lowerCamelCase,			attention_mask=_lowerCamelCase,			position_ids=_lowerCamelCase,			token_type_ids=_lowerCamelCase,			deterministic=_lowerCamelCase,			)
       __A		     =					text_outputs[1]
       __A		     =					module.text_projection(_lowerCamelCase    )
       return text_features

     return self.module.apply(
         {'''params''': params or self.params},			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			jnp.array(_lowerCamelCase,			dtype='''i4'''    ),			not train,			method=_get_features,			rngs=_lowerCamelCase,			)


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Union[str, Any],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	dict = None,			_lowerCamelCase					:	jax.random.PRNGKey = None,			_lowerCamelCase					:	int=False    ):

     '''simple docstring'''
     __A		     =					{}
     if dropout_rng is not None:
       __A		     =					dropout_rng

     def _get_features(_lowerCamelCase					:	Any,			_lowerCamelCase					:	Any,			_lowerCamelCase					:	Optional[Any]    ):
       __A		     =					module.vision_model(pixel_values=_lowerCamelCase,			deterministic=_lowerCamelCase    )
       __A		     =					vision_outputs[1]  # pooled_output
       __A		     =					module.visual_projection(_lowerCamelCase    )
       return image_features

     return self.module.apply(
         {'''params''': params or self.params},			jnp.array(_lowerCamelCase,			dtype=jnp.floataa    ),			not train,			method=_get_features,			rngs=_lowerCamelCase,			)






   @classmethod
   def 							_SCREAMING_SNAKE_CASE							(      cls					:	Dict,			_lowerCamelCase					:	str = None,			_lowerCamelCase					:	str = None,			*_lowerCamelCase					:	Optional[int],			**_lowerCamelCase					:	int,			):

     '''simple docstring'''
     __A		     =					{
         argument[len('''text_'''    ) :]: value for argument, value in kwargs.items() if argument.startswith('''text_'''    )
     }

     __A		     =					{
         argument[len('''vision_'''    ) :]: value for argument, value in kwargs.items() if argument.startswith('''vision_'''    )
     }

     # remove text, vision kwargs from kwargs
     for key in kwargs_text.keys():
       del kwargs["text_" + key]
     for key in kwargs_vision.keys():
       del kwargs["vision_" + key]

     # Load and initialize the text and vision model
     __A		     =					kwargs_text.pop('''model''',			_lowerCamelCase    )
     if text_model is None:
       assert (
           text_model_name_or_path is not None
       ), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
       from transformers import FlaxAutoModel

       if "config" not in kwargs_text:
         from transformers import AutoConfig

         __A		     =					AutoConfig.from_pretrained(_lowerCamelCase    )
         __A		     =					text_config

       __A		     =					FlaxAutoModel.from_pretrained(_lowerCamelCase,			*_lowerCamelCase,			**_lowerCamelCase    )

     __A		     =					kwargs_vision.pop('''model''',			_lowerCamelCase    )
     if vision_model is None:
       assert (
           vision_model_name_or_path is not None
       ), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
       from transformers import FlaxAutoModel

       if "config" not in kwargs_vision:
         from transformers import AutoConfig

         __A		     =					AutoConfig.from_pretrained(_lowerCamelCase    )
         __A		     =					vision_config

       __A		     =					FlaxAutoModel.from_pretrained(_lowerCamelCase,			*_lowerCamelCase,			**_lowerCamelCase    )

     # instantiate config with corresponding kwargs
     __A		     =					kwargs.pop('''dtype''',			jnp.floataa    )
     __A		     =					HybridCLIPConfig.from_text_vision_configs(text_model.config,			vision_model.config,			**_lowerCamelCase    )

     # init model
     __A		     =					cls(_lowerCamelCase,			*_lowerCamelCase,			dtype=_lowerCamelCase,			**_lowerCamelCase    )

     if vision_config.model_type == "clip":
       __A		     =					vision_model.params['''vision_model''']
       __A		     =					vision_model.params['''visual_projection''']['''kernel''']
     else:
       __A		     =					vision_model.params

     __A		     =					text_model.params

     return model





'''simple docstring'''





from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


UpperCamelCase       :     Any			=    logging.get_logger(__name__)

UpperCamelCase       :     Tuple			=    {
    """google/long-t5-local-base""": """https://huggingface.co/google/long-t5-local-base/blob/main/config.json""",
    """google/long-t5-local-large""": """https://huggingface.co/google/long-t5-local-large/blob/main/config.json""",
    """google/long-t5-tglobal-base""": """https://huggingface.co/google/long-t5-tglobal-base/blob/main/config.json""",
    """google/long-t5-tglobal-large""": """https://huggingface.co/google/long-t5-tglobal-large/blob/main/config.json""",
}



class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    A     :    Union[str, Any]				=			"longt5"
    A     :    List[str]				=			["past_key_values"]
    A     :    Dict				=			{"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}





    def __init__(					self     :						int							,							UpperCAmelCase_     :						Union[str, Any]=3_2_1_2_8							,							UpperCAmelCase_     :						List[Any]=5_1_2							,							UpperCAmelCase_     :						List[Any]=6_4							,							UpperCAmelCase_     :						List[str]=2_0_4_8							,							UpperCAmelCase_     :						List[str]=6							,							UpperCAmelCase_     :						Tuple=None							,							UpperCAmelCase_     :						Tuple=8							,							UpperCAmelCase_     :						str=1_2_7							,							UpperCAmelCase_     :						Dict=1_6							,							UpperCAmelCase_     :						Optional[int]=3_2							,							UpperCAmelCase_     :						Union[str, Any]=1_2_8							,							UpperCAmelCase_     :						str=0.1							,							UpperCAmelCase_     :						Optional[int]=1e-6							,							UpperCAmelCase_     :						Tuple=1.0							,							UpperCAmelCase_     :						str="relu"							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						Dict="local"							,							UpperCAmelCase_     :						Optional[int]=True							,							UpperCAmelCase_     :						Dict=0							,							UpperCAmelCase_     :						Union[str, Any]=1							,							**UpperCAmelCase_     :						Any							,							):


         """simple docstring"""







         a					:							Optional[int]							= vocab_size
         a					:							Tuple							= d_model
         a					:							str							= d_kv
         a					:							str							= d_ff
         a					:							Union[str, Any]							= num_layers
         # default = symmetry
         a					:							Optional[Any]							= num_decoder_layers if num_decoder_layers is not None else self.num_layers
         a					:							List[Any]							= num_heads
         a					:							List[str]							= local_radius
         a					:							Optional[Any]							= global_block_size
         a					:							Optional[int]							= relative_attention_num_buckets
         a					:							List[Any]							= relative_attention_max_distance
         a					:							int							= dropout_rate
         a					:							List[str]							= layer_norm_epsilon
         a					:							Optional[Any]							= initializer_factor
         a					:							List[Any]							= feed_forward_proj
         a					:							Dict							= encoder_attention_type
         a					:							int							= use_cache

         a					:							Optional[Any]							= self.feed_forward_proj.split('-')
         a					:							List[Any]							= act_info[-1]
         a					:							str							= act_info[0] == 'gated'

         if len(UpperCAmelCase_) > 1 and act_info[0] != "gated" or len(UpperCAmelCase_) > 2:
              raise ValueError(
                  f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
                  'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
                  '\'gated-gelu\' or \'relu\'')

         # for backwards compatibility
         if feed_forward_proj == "gated-gelu":
              a					:							Any							= 'gelu_new'

         super().__init__(
             pad_token_id=UpperCAmelCase_							,							eos_token_id=UpperCAmelCase_							,							is_encoder_decoder=UpperCAmelCase_							,							**UpperCAmelCase_							,							)






class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    @property
    def    SCREAMING_SNAKE_CASE_     (					self     :						Union[str, Any]):


         """simple docstring"""







         a					:							List[Any]							= {
             'input_ids': {0: 'batch', 1: 'encoder_sequence'},
             'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
         }
         if self.use_past:
              a					:							List[str]							= 'past_encoder_sequence + sequence'
              a					:							str							= {0: 'batch'}
              a					:							List[Any]							= {0: 'batch', 1: 'past_decoder_sequence + sequence'}
         else:
              a					:							Optional[Any]							= {0: 'batch', 1: 'decoder_sequence'}
              a					:							str							= {0: 'batch', 1: 'decoder_sequence'}

         if self.use_past:
              self.fill_with_past_key_values_(UpperCAmelCase_							,							direction='inputs')

         return common_inputs







    @property
    def    SCREAMING_SNAKE_CASE_     (					self     :						Union[str, Any]):


         """simple docstring"""







         return 1_3

