
import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


_A							=       [nn.LayerNorm]

_A							=       logging.get_logger(__name__)

_A							=       version.parse(version.parse(torch.__version__).base_version)

_A							=       parsed_torch_version_base >= version.parse('''2.0''')
_A							=       parsed_torch_version_base >= version.parse('''1.12''')
_A							=       parsed_torch_version_base >= version.parse('''1.11''')
_A							=       parsed_torch_version_base < version.parse('''1.11''')




def 					lowerCamelCase__					(    a__	:	int						,   a__	:	Union[str, Any]						,   a__	:	List[str]						,   a__	:	Any						,   self	:	Optional[int]  )		->						int:
       from torch import _softmax_backward_data

       if is_torch_less_than_1_11:
              return _softmax_backward_data(a_						,   a_						,   parent.dim						,   self  )
       else:
              return _softmax_backward_data(a_						,   a_						,   parent.dim						,   self.dtype  )




def 					lowerCamelCase__					(    a__	:	nn.Linear						,   a__	:	torch.LongTensor						,   a__	:	int = 0  )		->						nn.Linear:
       UpperCamelCase_											=							index.to(layer.weight.device  )
       UpperCamelCase_											=							layer.weight.index_select(a_						,   a_  ).clone().detach()
       if layer.bias is not None:
              if dim == 1:
                     UpperCamelCase_											=							layer.bias.clone().detach()
              else:
                     UpperCamelCase_											=							layer.bias[index].clone().detach()
       UpperCamelCase_											=							list(layer.weight.size()  )
       UpperCamelCase_											=							len(a_  )
       UpperCamelCase_											=							nn.Linear(new_size[1]						,   new_size[0]						,   bias=layer.bias is not None  ).to(layer.weight.device  )
       UpperCamelCase_											=							False
       new_layer.weight.copy_(W.contiguous()  )
       UpperCamelCase_											=							True
       if layer.bias is not None:
              UpperCamelCase_											=							False
              new_layer.bias.copy_(b.contiguous()  )
              UpperCamelCase_											=							True
       return new_layer

class 		lowercase_							(							nn.Module					):

 def __init__(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ):




        """simple docstring"""


        super().__init__()
        UpperCamelCase_											=							nf
        UpperCamelCase_											=							nn.Parameter(torch.empty(__UpperCamelCase  ,						__UpperCamelCase  )  )
        UpperCamelCase_											=							nn.Parameter(torch.zeros(__UpperCamelCase  )  )
        nn.init.normal_(self.weight  ,						std=0.02  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							x.size()[:-1] + (self.nf,)
        UpperCamelCase_											=							torch.addmm(self.bias  ,						x.view(-1  ,						x.size(-1  )  )  ,						self.weight  )
        UpperCamelCase_											=							x.view(__UpperCamelCase  )
        return x




def 					lowerCamelCase__					(    a__	:	ConvaD						,   a__	:	torch.LongTensor						,   a__	:	int = 1  )		->						ConvaD:
       UpperCamelCase_											=							index.to(layer.weight.device  )
       UpperCamelCase_											=							layer.weight.index_select(a_						,   a_  ).clone().detach()
       if dim == 0:
              UpperCamelCase_											=							layer.bias.clone().detach()
       else:
              UpperCamelCase_											=							layer.bias[index].clone().detach()
       UpperCamelCase_											=							list(layer.weight.size()  )
       UpperCamelCase_											=							len(a_  )
       UpperCamelCase_											=							ConvaD(new_size[1]						,   new_size[0]  ).to(layer.weight.device  )
       UpperCamelCase_											=							False
       new_layer.weight.copy_(W.contiguous()  )
       UpperCamelCase_											=							True
       UpperCamelCase_											=							False
       new_layer.bias.copy_(b.contiguous()  )
       UpperCamelCase_											=							True
       return new_layer




def 					lowerCamelCase__					(    a__	:	Union[nn.Linear, ConvaD]						,   a__	:	torch.LongTensor						,   a__	:	Optional[int] = None  )		->						Union[nn.Linear, ConvaD]:







       if isinstance(a_						,   nn.Linear  ):
              return prune_linear_layer(a_						,   a_						,   dim=0 if dim is None else dim  )
       elif isinstance(a_						,   a_  ):
              return prune_convad_layer(a_						,   a_						,   dim=1 if dim is None else dim  )
       else:
              raise ValueError(f'''Can\'t prune layer of class {layer.__class__}'''  )




def 					lowerCamelCase__					(    a__	:	Callable[..., torch.Tensor]						,   a__	:	int						,   a__	:	int						,   *a__	:	Union[str, Any]  )		->						torch.Tensor:
       assert len(a_  ) > 0, f'''{input_tensors} has to be a tuple/list of tensors'''

       # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
       UpperCamelCase_											=							len(inspect.signature(a_  ).parameters  )
       if num_args_in_forward_chunk_fn != len(a_  ):
              raise ValueError(
                  f'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(a_  )} input '''
                  """tensors are given"""  )

       if chunk_size > 0:
              UpperCamelCase_											=							input_tensors[0].shape[chunk_dim]
              for input_tensor in input_tensors:
                     if input_tensor.shape[chunk_dim] != tensor_shape:
                            raise ValueError(
                                f'''All input tenors have to be of the same shape: {tensor_shape}, '''
                                f'''found shape {input_tensor.shape[chunk_dim]}'''  )

              if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
                     raise ValueError(
                         f'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
                         f'''size {chunk_size}'''  )

              UpperCamelCase_											=							input_tensors[0].shape[chunk_dim] // chunk_size

              # chunk input tensor into tuples
              UpperCamelCase_											=							tuple(input_tensor.chunk(a_						,   dim=a_  ) for input_tensor in input_tensors  )
              # apply forward fn to every tuple
              UpperCamelCase_											=							tuple(forward_fn(*a_  ) for input_tensors_chunk in zip(*a_  )  )
              # concatenate output at same dimension
              return torch.cat(a_						,   dim=a_  )

       return forward_fn(*a_  )




def 					lowerCamelCase__					(    a__	:	List[int]						,   a__	:	int						,   a__	:	int						,   a__	:	Set[int]  )		->						Tuple[Set[int], torch.LongTensor]:
       UpperCamelCase_											=							torch.ones(a_						,   a_  )
       UpperCamelCase_											=							set(a_  ) - already_pruned_heads  # Convert to set and remove already pruned heads
       for head in heads:
              # Compute how many pruned heads are before the head and move the index accordingly
              UpperCamelCase_											=							head - sum(1 if h < head else 0 for h in already_pruned_heads  )
              UpperCamelCase_											=							0
       UpperCamelCase_											=							mask.view(-1  ).contiguous().eq(1  )
       UpperCamelCase_											=							torch.arange(len(a_  )  )[mask].long()
       return heads, index




def 					lowerCamelCase__					(    *a__	:	Union[torch.Tensor, List[torch.Tensor]]						,   a__	:	Optional[str] = None  )		->						Tuple[torch.Tensor, ...]:
       return torch.meshgrid(*a_						,   indexing=a_  )







def 					lowerCamelCase__					(    a__	:	torch.Tensor  )		->						Tuple[torch.device, int, int]:
       return tensor.device, storage_ptr(a_  ), storage_size(a_  )






from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__							=				logging.get_logger(__name__)

A__							=				{
    "microsoft/prophetnet-large-uncased": (
        "https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json"
    ),
}

class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):
	_lowercase	: Union[str, Any]												=	"prophetnet"
	_lowercase	: Any												=	["past_key_values"]
	_lowercase	: int												=	{
	    "num_attention_heads": "num_encoder_attention_heads",
	}







	def __init__( self:			Union[str, Any]	,			__UpperCamelCase:			Optional[float] = 0.1	,			__UpperCamelCase:			Optional[Union[str, Callable]] = "gelu"	,			__UpperCamelCase:			Optional[int] = 3_05_22	,			__UpperCamelCase:			Optional[int] = 10_24	,			__UpperCamelCase:			Optional[int] = 40_96	,			__UpperCamelCase:			Optional[int] = 12	,			__UpperCamelCase:			Optional[int] = 16	,			__UpperCamelCase:			Optional[int] = 40_96	,			__UpperCamelCase:			Optional[int] = 12	,			__UpperCamelCase:			Optional[int] = 16	,			__UpperCamelCase:			Optional[float] = 0.1	,			__UpperCamelCase:			Optional[float] = 0.1	,			__UpperCamelCase:			Optional[int] = 5_12	,			__UpperCamelCase:			Optional[float] = 0.02	,			__UpperCamelCase:			Optional[bool] = True	,			__UpperCamelCase:			Optional[bool] = True	,			__UpperCamelCase:			Optional[int] = 0	,			__UpperCamelCase:			Optional[int] = 2	,			__UpperCamelCase:			Optional[int] = 32	,			__UpperCamelCase:			Optional[int] = 1_28	,			__UpperCamelCase:			Optional[bool] = False	,			__UpperCamelCase:			Optional[float] = 0.0	,			__UpperCamelCase:			Optional[bool] = True	,			__UpperCamelCase:			Optional[int] = 0	,			__UpperCamelCase:			Optional[int] = 1	,			__UpperCamelCase:			Optional[int] = 2	,			**__UpperCamelCase:			Optional[int]	,			):

						'''simple docstring'''





						__magic_name__				=     vocab_size
						__magic_name__				=     hidden_size
						__magic_name__				=     encoder_ffn_dim
						__magic_name__				=     num_encoder_layers
						__magic_name__				=     num_encoder_attention_heads
						__magic_name__				=     decoder_ffn_dim
						__magic_name__				=     num_decoder_layers
						__magic_name__				=     num_decoder_attention_heads
						__magic_name__				=     max_position_embeddings
						__magic_name__				=     init_std  # Normal(0, this parameter)
						__magic_name__				=     activation_function

						# parameters for prophetnet
						__magic_name__				=     ngram
						__magic_name__				=     num_buckets
						__magic_name__				=     relative_max_distance
						__magic_name__				=     disable_ngram_loss
						__magic_name__				=     eps

						# 3 Types of Dropout
						__magic_name__				=     attention_dropout
						__magic_name__				=     activation_dropout
						__magic_name__				=     dropout

						__magic_name__				=     use_cache

						super().__init__(
						    pad_token_id=__UpperCamelCase	,			bos_token_id=__UpperCamelCase	,			eos_token_id=__UpperCamelCase	,			is_encoder_decoder=__UpperCamelCase	,			add_cross_attention=__UpperCamelCase	,			decoder_start_token_id=__UpperCamelCase	,			**__UpperCamelCase	,			)







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple   ):

						'''simple docstring'''





						return self.num_encoder_layers + self.num_decoder_layers







	@num_hidden_layers.setter
	def 						_SCREAMING_SNAKE_CASE				( self:			Union[str, Any]	,			__UpperCamelCase:			List[Any]   ):

						'''simple docstring'''





						raise NotImplementedError(
						    'This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'
						    ' `num_decoder_layers`.'   )




