




from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin



@dataclass
class       lowercase_		(A_   ):
						lowerCAmelCase__ 			=42



class       lowercase_		(A_       ,  A_   ):



						@register_to_config
						def __init__( self  :      Tuple				,       snake_case__  :      int = 16				,       snake_case__  :      int = 88				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      int = 1				,       snake_case__  :      float = 0.0				,       snake_case__  :      int = 32				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      bool = False				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      str = "geglu"				,       snake_case__  :      Optional[int] = None				,       snake_case__  :      bool = False				,       snake_case__  :      bool = False				,       snake_case__  :      bool = False				,       snake_case__  :      str = "layer_norm"				,       snake_case__  :      bool = True				,       ):

								"""simple docstring"""

								super().__init__()
								SCREAMING_SNAKE_CASE_							=       use_linear_projection
								SCREAMING_SNAKE_CASE_							=       num_attention_heads
								SCREAMING_SNAKE_CASE_							=       attention_head_dim
								SCREAMING_SNAKE_CASE_							=       num_attention_heads * attention_head_dim

								# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
								# Define whether input is continuous or discrete depending on configuration
								SCREAMING_SNAKE_CASE_							=       (in_channels is not None) and (patch_size is None)
								SCREAMING_SNAKE_CASE_							=       num_vector_embeds is not None
								SCREAMING_SNAKE_CASE_							=       in_channels is not None and patch_size is not None

								if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
										SCREAMING_SNAKE_CASE_							=       (
										    f'''The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'''
										    ''' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'''
										    ''' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'''
										    ''' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'''
										    ''' would be very nice if you could open a Pull request for the `transformer/config.json` file'''
										)
										deprecate('norm_type!=num_embeds_ada_norm'				,       '1.0.0'				,       snake_case__				,       standard_warn=snake_case__     )
										SCREAMING_SNAKE_CASE_							=       '''ada_norm'''

								if self.is_input_continuous and self.is_input_vectorized:
										raise ValueError(
										    f'''Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'''
										    ' sure that either `in_channels` or `num_vector_embeds` is None.'     )
								elif self.is_input_vectorized and self.is_input_patches:
										raise ValueError(
										    f'''Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'''
										    ' sure that either `num_vector_embeds` or `num_patches` is None.'     )
								elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
										raise ValueError(
										    f'''Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'''
										    f''' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'''     )

								# 2. Define input layers
								if self.is_input_continuous:
										SCREAMING_SNAKE_CASE_							=       in_channels

										SCREAMING_SNAKE_CASE_							=       torch.nn.GroupNorm(num_groups=snake_case__				,       num_channels=snake_case__				,       eps=1e-6				,       affine=snake_case__     )
										if use_linear_projection:
												SCREAMING_SNAKE_CASE_							=       nn.Linear(snake_case__				,       snake_case__     )
										else:
												SCREAMING_SNAKE_CASE_							=       nn.Convad(snake_case__				,       snake_case__				,       kernel_size=1				,       stride=1				,       padding=0     )
								elif self.is_input_vectorized:
										assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
										assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

										SCREAMING_SNAKE_CASE_							=       sample_size
										SCREAMING_SNAKE_CASE_							=       sample_size
										SCREAMING_SNAKE_CASE_							=       num_vector_embeds
										SCREAMING_SNAKE_CASE_							=       self.height * self.width

										SCREAMING_SNAKE_CASE_							=       ImagePositionalEmbeddings(
										    num_embed=snake_case__				,       embed_dim=snake_case__				,       height=self.height				,       width=self.width     )
								elif self.is_input_patches:
										assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

										SCREAMING_SNAKE_CASE_							=       sample_size
										SCREAMING_SNAKE_CASE_							=       sample_size

										SCREAMING_SNAKE_CASE_							=       patch_size
										SCREAMING_SNAKE_CASE_							=       PatchEmbed(
										    height=snake_case__				,       width=snake_case__				,       patch_size=snake_case__				,       in_channels=snake_case__				,       embed_dim=snake_case__				,       )

								# 3. Define transformers blocks
								SCREAMING_SNAKE_CASE_							=       nn.ModuleList(
								    [
								        BasicTransformerBlock(
								            snake_case__				,       snake_case__				,       snake_case__				,       dropout=snake_case__				,       cross_attention_dim=snake_case__				,       activation_fn=snake_case__				,       num_embeds_ada_norm=snake_case__				,       attention_bias=snake_case__				,       only_cross_attention=snake_case__				,       upcast_attention=snake_case__				,       norm_type=snake_case__				,       norm_elementwise_affine=snake_case__				,       )
								        for d in range(snake_case__     )
								    ]     )

								# 4. Define output layers
								SCREAMING_SNAKE_CASE_							=       in_channels if out_channels is None else out_channels
								if self.is_input_continuous:
										# TODO: should use out_channels for continuous projections
										if use_linear_projection:
												SCREAMING_SNAKE_CASE_							=       nn.Linear(snake_case__				,       snake_case__     )
										else:
												SCREAMING_SNAKE_CASE_							=       nn.Convad(snake_case__				,       snake_case__				,       kernel_size=1				,       stride=1				,       padding=0     )
								elif self.is_input_vectorized:
										SCREAMING_SNAKE_CASE_							=       nn.LayerNorm(snake_case__     )
										SCREAMING_SNAKE_CASE_							=       nn.Linear(snake_case__				,       self.num_vector_embeds - 1     )
								elif self.is_input_patches:
										SCREAMING_SNAKE_CASE_							=       nn.LayerNorm(snake_case__				,       elementwise_affine=snake_case__				,       eps=1e-6     )
										SCREAMING_SNAKE_CASE_							=       nn.Linear(snake_case__				,       2 * inner_dim     )
										SCREAMING_SNAKE_CASE_							=       nn.Linear(snake_case__				,       patch_size * patch_size * self.out_channels     )






						def   __a				( self  :      List[Any]				,       snake_case__  :      torch.Tensor				,       snake_case__  :      Optional[torch.Tensor] = None				,       snake_case__  :      Optional[torch.LongTensor] = None				,       snake_case__  :      Optional[torch.LongTensor] = None				,       snake_case__  :      Dict[str, Any] = None				,       snake_case__  :      Optional[torch.Tensor] = None				,       snake_case__  :      Optional[torch.Tensor] = None				,       snake_case__  :      bool = True				,       ):

								"""simple docstring"""

								if attention_mask is not None and attention_mask.ndim == 2:
										# assume that mask is expressed as:
										#   (1 = keep,      0 = discard)
										# convert mask into a bias that can be added to attention scores:
										#       (keep = +0,     discard = -10000.0)
										SCREAMING_SNAKE_CASE_							=       (1 - attention_mask.to(hidden_states.dtype     )) * -1_00_00.0
										SCREAMING_SNAKE_CASE_							=       attention_mask.unsqueeze(1     )

								# convert encoder_attention_mask to a bias the same way we do for attention_mask
								if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
										SCREAMING_SNAKE_CASE_							=       (1 - encoder_attention_mask.to(hidden_states.dtype     )) * -1_00_00.0
										SCREAMING_SNAKE_CASE_							=       encoder_attention_mask.unsqueeze(1     )

								# 1. Input
								if self.is_input_continuous:
										SCREAMING_SNAKE_CASE_							=       hidden_states.shape
										SCREAMING_SNAKE_CASE_							=       hidden_states

										SCREAMING_SNAKE_CASE_							=       self.norm(snake_case__     )
										if not self.use_linear_projection:
												SCREAMING_SNAKE_CASE_							=       self.proj_in(snake_case__     )
												SCREAMING_SNAKE_CASE_							=       hidden_states.shape[1]
												SCREAMING_SNAKE_CASE_							=       hidden_states.permute(0				,       2				,       3				,       1     ).reshape(snake_case__				,       height * width				,       snake_case__     )
										else:
												SCREAMING_SNAKE_CASE_							=       hidden_states.shape[1]
												SCREAMING_SNAKE_CASE_							=       hidden_states.permute(0				,       2				,       3				,       1     ).reshape(snake_case__				,       height * width				,       snake_case__     )
												SCREAMING_SNAKE_CASE_							=       self.proj_in(snake_case__     )
								elif self.is_input_vectorized:
										SCREAMING_SNAKE_CASE_							=       self.latent_image_embedding(snake_case__     )
								elif self.is_input_patches:
										SCREAMING_SNAKE_CASE_							=       self.pos_embed(snake_case__     )

								# 2. Blocks
								for block in self.transformer_blocks:
										SCREAMING_SNAKE_CASE_							=       block(
										    snake_case__				,       attention_mask=snake_case__				,       encoder_hidden_states=snake_case__				,       encoder_attention_mask=snake_case__				,       timestep=snake_case__				,       cross_attention_kwargs=snake_case__				,       class_labels=snake_case__				,       )

								# 3. Output
								if self.is_input_continuous:
										if not self.use_linear_projection:
												SCREAMING_SNAKE_CASE_							=       hidden_states.reshape(snake_case__				,       snake_case__				,       snake_case__				,       snake_case__     ).permute(0				,       3				,       1				,       2     ).contiguous()
												SCREAMING_SNAKE_CASE_							=       self.proj_out(snake_case__     )
										else:
												SCREAMING_SNAKE_CASE_							=       self.proj_out(snake_case__     )
												SCREAMING_SNAKE_CASE_							=       hidden_states.reshape(snake_case__				,       snake_case__				,       snake_case__				,       snake_case__     ).permute(0				,       3				,       1				,       2     ).contiguous()

										SCREAMING_SNAKE_CASE_							=       hidden_states + residual
								elif self.is_input_vectorized:
										SCREAMING_SNAKE_CASE_							=       self.norm_out(snake_case__     )
										SCREAMING_SNAKE_CASE_							=       self.out(snake_case__     )
										# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
										SCREAMING_SNAKE_CASE_							=       logits.permute(0				,       2				,       1     )

										# log(p(x_0))
										SCREAMING_SNAKE_CASE_							=       F.log_softmax(logits.double()				,       dim=1     ).float()
								elif self.is_input_patches:
										# TODO: cleanup!
										SCREAMING_SNAKE_CASE_							=       self.transformer_blocks[0].norma.emb(
										    snake_case__				,       snake_case__				,       hidden_dtype=hidden_states.dtype     )
										SCREAMING_SNAKE_CASE_							=       self.proj_out_a(F.silu(snake_case__     )     ).chunk(2				,       dim=1     )
										SCREAMING_SNAKE_CASE_							=       self.norm_out(snake_case__     ) * (1 + scale[:, None]) + shift[:, None]
										SCREAMING_SNAKE_CASE_							=       self.proj_out_a(snake_case__     )

										# unpatchify
										SCREAMING_SNAKE_CASE_							=       int(hidden_states.shape[1] ** 0.5     )
										SCREAMING_SNAKE_CASE_							=       hidden_states.reshape(
										    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)     )
										SCREAMING_SNAKE_CASE_							=       torch.einsum('nhwpqc->nchpwq'				,       snake_case__     )
										SCREAMING_SNAKE_CASE_							=       hidden_states.reshape(
										    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)     )

								if not return_dict:
										return (output,)

								return TransformeraDModelOutput(sample=snake_case__     )





import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    SummarizationPipeline,
    TFPreTrainedModel,
    pipeline,
)
from transformers.testing_utils import get_gpu_count, is_pipeline_test, require_tf, require_torch, slow, torch_device
from transformers.tokenization_utils import TruncationStrategy

from .test_pipelines_common import ANY


SCREAMING_SNAKE_CASE:    Optional[Any]											=   -1 if torch_device == '''cpu''' else 0



@is_pipeline_test
class       lowercase_		(unittest.TestCase   ):
						lowerCAmelCase__ 			=MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
						lowerCAmelCase__ 			=TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING



						def   __a				( self  :      Optional[Any]				,       snake_case__  :      Tuple				,       snake_case__  :      Tuple				,       snake_case__  :      Optional[Any]     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       SummarizationPipeline(model=snake_case__				,       tokenizer=snake_case__     )
								return summarizer, ["(CNN)The Palestinian Authority officially became", "Some other text"]



						def   __a				( self  :      Tuple				,       snake_case__  :      str				,       snake_case__  :      Union[str, Any]     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       summarizer.model

								SCREAMING_SNAKE_CASE_							=       summarizer('(CNN)The Palestinian Authority officially became'     )
								self.assertEqual(snake_case__				,       [{'summary_text': ANY(snake_case__     )}]     )

								SCREAMING_SNAKE_CASE_							=       summarizer(
								    '(CNN)The Palestinian Authority officially became '				,       num_beams=2				,       min_length=2				,       max_length=5				,       )
								self.assertEqual(snake_case__				,       [{'summary_text': ANY(snake_case__     )}]     )

								# Some models (Switch Transformers, LED, T5, LongT5, etc) can handle long sequences.
								SCREAMING_SNAKE_CASE_							=       [
								    'SwitchTransformersConfig',
								    'T5Config',
								    'LongT5Config',
								    'LEDConfig',
								    'PegasusXConfig',
								    'FSMTConfig',
								    'M2M100Config',
								    'ProphetNetConfig',  # positional embeddings up to a fixed maximum size (otherwise clamping the values)
								]
								if model.config.__class__.__name__ not in model_can_handle_longer_seq:
										# Too long and exception is expected.
										# For TF models, if the weights are initialized in GPU context, we won't get expected index error from
										# the embedding layer.
										if not (
										    isinstance(snake_case__				,       snake_case__     )
										    and get_gpu_count() > 0
										    and len(summarizer.model.trainable_weights     ) > 0
										):
												with self.assertRaises(snake_case__     ):
														SCREAMING_SNAKE_CASE_							=       summarizer('This ' * 10_00     )
								SCREAMING_SNAKE_CASE_							=       summarizer('This ' * 10_00				,       truncation=TruncationStrategy.ONLY_FIRST     )



						@require_torch
						def   __a				( self  :      Dict     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       pipeline(task='summarization'				,       model='sshleifer/tiny-mbart'				,       framework='pt'     )
								SCREAMING_SNAKE_CASE_							=       summarizer('This is a small test'     )
								self.assertEqual(
								    snake_case__				,       [
								        {
								            'summary_text': 'เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป'
								        }
								    ]				,       )



						@require_tf
						def   __a				( self  :      Any     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       pipeline(task='summarization'				,       model='sshleifer/tiny-mbart'				,       framework='tf'     )
								SCREAMING_SNAKE_CASE_							=       summarizer('This is a small test'     )
								self.assertEqual(
								    snake_case__				,       [
								        {
								            'summary_text': 'เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป'
								        }
								    ]				,       )






						@require_torch
						@slow
						def   __a				( self  :      Any     ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_							=       pipeline(task='summarization'				,       device=snake_case__     )
								SCREAMING_SNAKE_CASE_							=       (
								    ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on'
								    ' Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The'
								    ' formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based.'
								    ' The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its'
								    ' jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East'
								    ' Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the'
								    ' situation in Palestinian territories, paving the way for possible war crimes investigations against'
								    ' Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and'
								    ' the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the'
								    ' body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a'
								    ' move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the'
								    ' world is also a step closer to ending a long era of impunity and injustice," he said, according to an'
								    ' ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge'
								    ' Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the'
								    ' Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine'
								    ' acquires all the rights as well as responsibilities that come with being a State Party to the Statute.'
								    ' These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights'
								    ' Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should'
								    ' immediately end their pressure, and countries that support universal acceptance of the court\'s treaty'
								    ' should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the'
								    ' group. \"What\'s objectionable is the attempts to undermine international justice, not Palestine\'s'
								    ' decision to join a treaty to which over 100 countries around the world are members." In January, when'
								    ' the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an'
								    ' outrage, saying the court was overstepping its boundaries. The United States also said it "strongly"'
								    ' disagreed with the court\'s decision. \"As we have said repeatedly, we do not believe that Palestine is a'
								    ' state and therefore we do not believe that it is eligible to join the ICC," the State Department said in'
								    ' a statement. It urged the warring sides to resolve their differences through direct negotiations. "We'
								    ' will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace,"'
								    ' it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the'
								    ' territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the'
								    ' court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou'
								    ' Bensouda said her office would "conduct its analysis in full independence and impartiality." The war'
								    ' between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry'
								    ' will include alleged war crimes committed since June. The International Criminal Court was set up in'
								    ' 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder'
								    ' and Faith Karimi contributed to this report.'
								)
								SCREAMING_SNAKE_CASE_							=       (
								    ' The Palestinian Authority becomes the 123rd member of the International Criminal Court . The move gives'
								    ' the court jurisdiction over alleged crimes in Palestinian territories . Israel and the United States'
								    ' opposed the Palestinians\' efforts to join the court . Rights group Human Rights Watch welcomes the move,'
								    ' says governments seeking to penalize Palestine should end pressure .'
								)
								SCREAMING_SNAKE_CASE_							=       summarizer(snake_case__     )
								self.assertEqual(result[0]['summary_text']				,       snake_case__     )