


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
	import torch

	from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer





@require_torch
class 					snake_case__  ( unittest.TestCase       ):





	@slow
	def     A				(  self							)		->			Union[str, Any]:
								"""simple docstring"""






								a_       :   Union[str, Any]			=	XLMProphetNetForConditionalGeneration.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""							)
								model.to(UpperCamelCase_							)

								# encoder-decoder outputs
								a_       :   List[str]			=	torch.tensor([[17, 96208, 103471, 2]]							).to(UpperCamelCase_							)
								a_       :   str			=	torch.tensor(
								    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]							).to(UpperCamelCase_							)
								a_       :   Optional[Any]			=	model(
								    input_ids=UpperCamelCase_				,    attention_mask=UpperCamelCase_				,    encoder_outputs=UpperCamelCase_				,    decoder_input_ids=UpperCamelCase_							)
								a_       :   Optional[int]			=	output[0]
								a_       :   Dict			=	torch.Size((1, 14, 250012)							)
								self.assertEqual(output_predited_logis.shape				,    UpperCamelCase_							)
								a_       :   Optional[Any]			=	torch.tensor(
								    [[[-6.3986, -8.2391, 12.5189], [-6.3289, -8.0864, 12.6211], [-6.2418, -8.0445, 12.7968]]]							).to(UpperCamelCase_							)
								self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]				,    UpperCamelCase_				,    atol=1e-4							)							)

								# encoder outputs
								a_       :   Union[str, Any]			=	model.prophetnet.encoder(UpperCamelCase_							)[0]
								a_       :   int			=	torch.tensor(
								    [[[-1.4260, -0.7628, 0.8453], [-1.4719, -0.1391, 0.7807], [-1.7678, 0.0114, 0.4646]]]							).to(UpperCamelCase_							)
								a_       :   Optional[Any]			=	torch.Size((1, 4, 1024)							)
								self.assertEqual(encoder_outputs.shape				,    UpperCamelCase_							)
								self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]				,    UpperCamelCase_				,    atol=1e-4							)							)

								# decoder outputs
								a_       :   Dict			=	model.prophetnet.decoder(
								    UpperCamelCase_				,    encoder_hidden_states=UpperCamelCase_				,    )
								a_       :   List[str]			=	decoder_outputs[1].view(1				,    model.config.ngram				,    14				,    -1							)
								a_       :   str			=	model.lm_head(UpperCamelCase_							)
								a_       :   Optional[int]			=	predicting_streams_logits[:, 0]
								self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]				,    UpperCamelCase_				,    atol=1e-4							)							)





	@slow
	def     A				(  self							)		->			Any:
								"""simple docstring"""






								a_       :   Union[str, Any]			=	XLMProphetNetForConditionalGeneration.from_pretrained(
								    """microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"""							)
								model.to(UpperCamelCase_							)

								a_       :   Optional[int]			=	torch.tensor([[17, 96208, 103471, 2]]							).to(UpperCamelCase_							)
								a_       :   str			=	torch.tensor(
								    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]							).to(UpperCamelCase_							)
								a_       :   Optional[Any]			=	model(
								    input_ids=UpperCamelCase_				,    attention_mask=UpperCamelCase_				,    encoder_outputs=UpperCamelCase_				,    decoder_input_ids=UpperCamelCase_							)
								a_       :   List[str]			=	output[0]
								a_       :   Dict			=	torch.Size((1, 14, 250012)							)
								self.assertEqual(output_predited_logis.shape				,    UpperCamelCase_							)
								# compare the actual values for a slice.
								a_       :   int			=	torch.tensor(
								    [[[-9.2253, -9.7173, -6.3529], [-7.6701, -9.0145, -1.9382], [-8.0195, -7.0004, -0.1523]]]							).to(UpperCamelCase_							)

								self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]				,    UpperCamelCase_				,    atol=1e-4							)							)





	@slow
	def     A				(  self							)		->			Optional[Any]:
								"""simple docstring"""






								a_       :   Optional[Any]			=	XLMProphetNetForConditionalGeneration.from_pretrained(
								    """microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"""							)
								model.to(UpperCamelCase_							)
								a_       :   str			=	512

								a_       :   List[str]			=	XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"""							)

								a_       :   Dict			=	(
								    """Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"""
								    """ January 14, 2020, according to the official portal of the organization. From that day, users of this"""
								    """ system will not be able to receive security updates, which could make their computers vulnerable to"""
								    """ cyber attacks."""
								)
								a_       :   Optional[int]			=	(
								    """орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"""
								    """ после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"""
								    """ этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"""
								    """ к кибератакам."""
								)
								a_       :   List[Any]			=	(
								    """根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"""
								)

								a_       :   Union[str, Any]			=	tokenizer(
								    [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]				,    padding=UpperCamelCase_				,    max_length=255				,    return_tensors="""pt"""							).input_ids
								a_       :   int			=	input_ids.to(UpperCamelCase_							)

								a_       :   int			=	model.generate(
								    UpperCamelCase_				,    num_beams=10				,    length_penalty=1.0				,    no_repeat_ngram_size=3				,    early_stopping=UpperCamelCase_							)
								a_       :   Optional[int]			=	[tokenizer.decode(UpperCamelCase_				,    skip_special_tokens=UpperCamelCase_							) for g in summary_ids]
								a_       :   Optional[int]			=	"""Microsoft to end Windows 7 free support after January 14, 2020"""
								a_       :   Dict			=	"""Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"""
								a_       :   str			=	"""微软打算终止对Windows 7操作系统的免费支持"""
								self.assertListEqual(
								    [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]				,    UpperCamelCase_				,    )

								a_       :   List[str]			=	model.generate(
								    UpperCamelCase_				,    num_beams=1				,    length_penalty=1.0				,    no_repeat_ngram_size=3				,    early_stopping=UpperCamelCase_							)
								a_       :   List[str]			=	[
								    tokenizer.convert_ids_to_tokens(UpperCamelCase_				,    skip_special_tokens=UpperCamelCase_							) for g in summary_ids_beama
								]
								a_       :   List[str]			=	"""▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7""".split(""" """							)
								a_       :   Tuple			=	"""▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года""".split(
								    """ """							)
								a_       :   List[str]			=	"""微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持""".split(""" """							)
								self.assertListEqual(
								    [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]				,    UpperCamelCase_				,    )






from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin





@dataclass
class 					snake_case__  ( __A       ):
	UpperCAmelCase   :							torch.FloatTensor





class 					snake_case__  ( __A      ,    __A       ):





	@register_to_config
	def __init__(  self				,    UpperCamelCase_ = 16				,    UpperCamelCase_ = 88				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = 1				,    UpperCamelCase_ = 0.0				,    UpperCamelCase_ = 32				,    UpperCamelCase_ = None				,    UpperCamelCase_ = False				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = "geglu"				,    UpperCamelCase_ = None				,    UpperCamelCase_ = False				,    UpperCamelCase_ = False				,    UpperCamelCase_ = False				,    UpperCamelCase_ = "layer_norm"				,    UpperCamelCase_ = True				,    )		->			Union[str, Any]:
								"""simple docstring"""






								super().__init__()
								a_       :   Dict			=	use_linear_projection
								a_       :   Tuple			=	num_attention_heads
								a_       :   List[Any]			=	attention_head_dim
								a_       :   Dict			=	num_attention_heads * attention_head_dim

								# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
								# Define whether input is continuous or discrete depending on configuration
								a_       :   List[Any]			=	(in_channels is not None) and (patch_size is None)
								a_       :   Dict			=	num_vector_embeds is not None
								a_       :   Union[str, Any]			=	in_channels is not None and patch_size is not None

								if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
															a_       :   int			=	(
															    f"""The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"""
															    """ incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."""
															    """ Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"""
															    """ results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"""
															    """ would be very nice if you could open a Pull request for the `transformer/config.json` file"""
															)
															deprecate("""norm_type!=num_embeds_ada_norm"""				,    """1.0.0"""				,    UpperCamelCase_				,    standard_warn=UpperCamelCase_							)
															a_       :   List[Any]			=	"""ada_norm"""

								if self.is_input_continuous and self.is_input_vectorized:
															raise ValueError(
															    f"""Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"""
															    """ sure that either `in_channels` or `num_vector_embeds` is None."""							)
								elif self.is_input_vectorized and self.is_input_patches:
															raise ValueError(
															    f"""Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"""
															    """ sure that either `num_vector_embeds` or `num_patches` is None."""							)
								elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
															raise ValueError(
															    f"""Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"""
															    f""" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."""							)

								# 2. Define input layers
								if self.is_input_continuous:
															a_       :   List[Any]			=	in_channels

															a_       :   List[str]			=	torch.nn.GroupNorm(num_groups=UpperCamelCase_				,    num_channels=UpperCamelCase_				,    eps=1e-6				,    affine=UpperCamelCase_							)
															if use_linear_projection:
																						a_       :   int			=	nn.Linear(UpperCamelCase_				,    UpperCamelCase_							)
															else:
																						a_       :   Optional[Any]			=	nn.Convad(UpperCamelCase_				,    UpperCamelCase_				,    kernel_size=1				,    stride=1				,    padding=0							)
								elif self.is_input_vectorized:
															assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
															assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

															a_       :   Optional[Any]			=	sample_size
															a_       :   str			=	sample_size
															a_       :   str			=	num_vector_embeds
															a_       :   str			=	self.height * self.width

															a_       :   int			=	ImagePositionalEmbeddings(
															    num_embed=UpperCamelCase_				,    embed_dim=UpperCamelCase_				,    height=self.height				,    width=self.width							)
								elif self.is_input_patches:
															assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

															a_       :   Dict			=	sample_size
															a_       :   List[Any]			=	sample_size

															a_       :   Tuple			=	patch_size
															a_       :   Optional[Any]			=	PatchEmbed(
															    height=UpperCamelCase_				,    width=UpperCamelCase_				,    patch_size=UpperCamelCase_				,    in_channels=UpperCamelCase_				,    embed_dim=UpperCamelCase_				,    )

								# 3. Define transformers blocks
								a_       :   List[Any]			=	nn.ModuleList(
								    [
								        BasicTransformerBlock(
								            UpperCamelCase_				,    UpperCamelCase_				,    UpperCamelCase_				,    dropout=UpperCamelCase_				,    cross_attention_dim=UpperCamelCase_				,    activation_fn=UpperCamelCase_				,    num_embeds_ada_norm=UpperCamelCase_				,    attention_bias=UpperCamelCase_				,    only_cross_attention=UpperCamelCase_				,    upcast_attention=UpperCamelCase_				,    norm_type=UpperCamelCase_				,    norm_elementwise_affine=UpperCamelCase_				,    )
								        for d in range(UpperCamelCase_							)
								    ]							)

								# 4. Define output layers
								a_       :   str			=	in_channels if out_channels is None else out_channels
								if self.is_input_continuous:
															# TODO: should use out_channels for continuous projections
															if use_linear_projection:
																						a_       :   Optional[int]			=	nn.Linear(UpperCamelCase_				,    UpperCamelCase_							)
															else:
																						a_       :   Any			=	nn.Convad(UpperCamelCase_				,    UpperCamelCase_				,    kernel_size=1				,    stride=1				,    padding=0							)
								elif self.is_input_vectorized:
															a_       :   Optional[int]			=	nn.LayerNorm(UpperCamelCase_							)
															a_       :   str			=	nn.Linear(UpperCamelCase_				,    self.num_vector_embeds - 1							)
								elif self.is_input_patches:
															a_       :   int			=	nn.LayerNorm(UpperCamelCase_				,    elementwise_affine=UpperCamelCase_				,    eps=1e-6							)
															a_       :   str			=	nn.Linear(UpperCamelCase_				,    2 * inner_dim							)
															a_       :   Optional[int]			=	nn.Linear(UpperCamelCase_				,    patch_size * patch_size * self.out_channels							)





	def     A				(  self				,    UpperCamelCase_				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = None				,    UpperCamelCase_ = True				,    )		->			str:
								"""simple docstring"""






								if attention_mask is not None and attention_mask.ndim == 2:
															# assume that mask is expressed as:
															#   (1 = keep,      0 = discard)
															# convert mask into a bias that can be added to attention scores:
															#       (keep = +0,     discard = -10000.0)
															a_       :   str			=	(1 - attention_mask.to(hidden_states.dtype							)) * -1_0000.0
															a_       :   Union[str, Any]			=	attention_mask.unsqueeze(1							)

								# convert encoder_attention_mask to a bias the same way we do for attention_mask
								if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
															a_       :   Any			=	(1 - encoder_attention_mask.to(hidden_states.dtype							)) * -1_0000.0
															a_       :   List[str]			=	encoder_attention_mask.unsqueeze(1							)

								# 1. Input
								if self.is_input_continuous:
															a_		,				a_		,				a_		,				a_       :   str			=	hidden_states.shape
															a_       :   Optional[Any]			=	hidden_states

															a_       :   Optional[int]			=	self.norm(UpperCamelCase_							)
															if not self.use_linear_projection:
																						a_       :   int			=	self.proj_in(UpperCamelCase_							)
																						a_       :   str			=	hidden_states.shape[1]
																						a_       :   List[str]			=	hidden_states.permute(0				,    2				,    3				,    1							).reshape(UpperCamelCase_				,    height * width				,    UpperCamelCase_							)
															else:
																						a_       :   int			=	hidden_states.shape[1]
																						a_       :   Optional[int]			=	hidden_states.permute(0				,    2				,    3				,    1							).reshape(UpperCamelCase_				,    height * width				,    UpperCamelCase_							)
																						a_       :   Tuple			=	self.proj_in(UpperCamelCase_							)
								elif self.is_input_vectorized:
															a_       :   List[str]			=	self.latent_image_embedding(UpperCamelCase_							)
								elif self.is_input_patches:
															a_       :   Optional[int]			=	self.pos_embed(UpperCamelCase_							)

								# 2. Blocks
								for block in self.transformer_blocks:
															a_       :   int			=	block(
															    UpperCamelCase_				,    attention_mask=UpperCamelCase_				,    encoder_hidden_states=UpperCamelCase_				,    encoder_attention_mask=UpperCamelCase_				,    timestep=UpperCamelCase_				,    cross_attention_kwargs=UpperCamelCase_				,    class_labels=UpperCamelCase_				,    )

								# 3. Output
								if self.is_input_continuous:
															if not self.use_linear_projection:
																						a_       :   Dict			=	hidden_states.reshape(UpperCamelCase_				,    UpperCamelCase_				,    UpperCamelCase_				,    UpperCamelCase_							).permute(0				,    3				,    1				,    2							).contiguous()
																						a_       :   List[Any]			=	self.proj_out(UpperCamelCase_							)
															else:
																						a_       :   Optional[int]			=	self.proj_out(UpperCamelCase_							)
																						a_       :   int			=	hidden_states.reshape(UpperCamelCase_				,    UpperCamelCase_				,    UpperCamelCase_				,    UpperCamelCase_							).permute(0				,    3				,    1				,    2							).contiguous()

															a_       :   int			=	hidden_states + residual
								elif self.is_input_vectorized:
															a_       :   str			=	self.norm_out(UpperCamelCase_							)
															a_       :   str			=	self.out(UpperCamelCase_							)
															# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
															a_       :   int			=	logits.permute(0				,    2				,    1							)

															# log(p(x_0))
															a_       :   int			=	F.log_softmax(logits.double()				,    dim=1							).float()
								elif self.is_input_patches:
															# TODO: cleanup!
															a_       :   Union[str, Any]			=	self.transformer_blocks[0].norma.emb(
															    UpperCamelCase_				,    UpperCamelCase_				,    hidden_dtype=hidden_states.dtype							)
															a_		,				a_       :   int			=	self.proj_out_a(F.silu(UpperCamelCase_							)							).chunk(2				,    dim=1							)
															a_       :   Union[str, Any]			=	self.norm_out(UpperCamelCase_							) * (1 + scale[:, None]) + shift[:, None]
															a_       :   Union[str, Any]			=	self.proj_out_a(UpperCamelCase_							)

															# unpatchify
															a_       :   Any			=	int(hidden_states.shape[1] ** 0.5							)
															a_       :   str			=	hidden_states.reshape(
															    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)							)
															a_       :   Union[str, Any]			=	torch.einsum("""nhwpqc->nchpwq"""				,    UpperCamelCase_							)
															a_       :   List[Any]			=	hidden_states.reshape(
															    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)							)

								if not return_dict:
															return (output,)

								return TransformeraDModelOutput(sample=UpperCamelCase_							)


