code: 
  
  
  from typing import Callable, Optional, Union
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  UpperCamelCase__				       =			logging.get_logger(__name__)
  
  UpperCamelCase__				       =			{
      'microsoft/prophetnet-large-uncased': (
          'https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json'
      ),
  }
  
  class 						UpperCAmelCase__   (      A_		):
  
  
  
       '''simple docstring'''
       UpperCAmelCase_         =  '''prophetnet'''
       UpperCAmelCase_         =  ['''past_key_values''']
       UpperCAmelCase_         =  {
           '''num_attention_heads''': '''num_encoder_attention_heads''',
       }
  
  
  
  
       def __init__( self       :       Optional[int]	,      UpperCamelCase       :       Optional[float] = 0.1	,      UpperCamelCase       :       Optional[Union[str, Callable]] = "gelu"	,      UpperCamelCase       :       Optional[int] = 3_05_22	,      UpperCamelCase       :       Optional[int] = 10_24	,      UpperCamelCase       :       Optional[int] = 40_96	,      UpperCamelCase       :       Optional[int] = 12	,      UpperCamelCase       :       Optional[int] = 16	,      UpperCamelCase       :       Optional[int] = 40_96	,      UpperCamelCase       :       Optional[int] = 12	,      UpperCamelCase       :       Optional[int] = 16	,      UpperCamelCase       :       Optional[float] = 0.1	,      UpperCamelCase       :       Optional[float] = 0.1	,      UpperCamelCase       :       Optional[int] = 5_12	,      UpperCamelCase       :       Optional[float] = 0.02	,      UpperCamelCase       :       Optional[bool] = True	,      UpperCamelCase       :       Optional[bool] = True	,      UpperCamelCase       :       Optional[int] = 0	,      UpperCamelCase       :       Optional[int] = 2	,      UpperCamelCase       :       Optional[int] = 32	,      UpperCamelCase       :       Optional[int] = 1_28	,      UpperCamelCase       :       Optional[bool] = False	,      UpperCamelCase       :       Optional[float] = 0.0	,      UpperCamelCase       :       Optional[bool] = True	,      UpperCamelCase       :       Optional[int] = 0	,      UpperCamelCase       :       Optional[int] = 1	,      UpperCamelCase       :       Optional[int] = 2	,      **UpperCamelCase       :       List[Any]	,      ):
  
  
  
  
  
              """simple docstring"""
  
  
  
              _lowercase					:				Optional[int]           =			vocab_size
              _lowercase					:				Tuple           =			hidden_size
              _lowercase					:				Union[str, Any]           =			encoder_ffn_dim
              _lowercase					:				Any           =			num_encoder_layers
              _lowercase					:				Dict           =			num_encoder_attention_heads
              _lowercase					:				Any           =			decoder_ffn_dim
              _lowercase					:				int           =			num_decoder_layers
              _lowercase					:				str           =			num_decoder_attention_heads
              _lowercase					:				List[str]           =			max_position_embeddings
              _lowercase					:				Any           =			init_std  # Normal(0, this parameter)
              _lowercase					:				Tuple           =			activation_function
  
              # parameters for prophetnet
              _lowercase					:				Dict           =			ngram
              _lowercase					:				Union[str, Any]           =			num_buckets
              _lowercase					:				Dict           =			relative_max_distance
              _lowercase					:				Optional[Any]           =			disable_ngram_loss
              _lowercase					:				Union[str, Any]           =			eps
  
              # 3 Types of Dropout
              _lowercase					:				Any           =			attention_dropout
              _lowercase					:				Optional[Any]           =			activation_dropout
              _lowercase					:				str           =			dropout
  
              _lowercase					:				int           =			use_cache
  
              super().__init__(
                  pad_token_id=UpperCamelCase	,      bos_token_id=UpperCamelCase	,      eos_token_id=UpperCamelCase	,      is_encoder_decoder=UpperCamelCase	,      add_cross_attention=UpperCamelCase	,      decoder_start_token_id=UpperCamelCase	,      **UpperCamelCase	,      )
  
  
  
  
       @property
       def   lowerCAmelCase_   ( self       :       Optional[int]		):
  
  
  
  
  
              """simple docstring"""
  
  
  
              return self.num_encoder_layers + self.num_decoder_layers
  
  
  
  
       @num_hidden_layers.setter
       def   lowerCAmelCase_   ( self       :       int	,      UpperCamelCase       :       str		):
  
  
  
  
  
              """simple docstring"""
  
  
  
              raise NotImplementedError(
                  '''This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'''
                  ''' `num_decoder_layers`.'''		)
style_context: 
  
  
  import torch
  import torch.nn as nn
  from transformers.modeling_utils import ModuleUtilsMixin
  from transformers.models.ta.modeling_ta import (
      TaBlock,
      TaConfig,
      TaLayerNorm,
  )
  
  from ...configuration_utils import ConfigMixin, register_to_config
  from ...models import ModelMixin
  
  class 						UpperCAmelCase__   (      A_      ,						A_      ,						A_		):
  
  
  
       '''simple docstring'''
  
  
  
  
       @register_to_config
       def __init__( self       :       Dict	,      UpperCamelCase       :       int	,      UpperCamelCase       :       int	,      UpperCamelCase       :       int	,      UpperCamelCase       :       float	,      UpperCamelCase       :       int	,      UpperCamelCase       :       int	,      UpperCamelCase       :       int	,      UpperCamelCase       :       int	,      UpperCamelCase       :       str	,      UpperCamelCase       :       bool = False	,      ):
  
  
  
  
  
              """simple docstring"""
  
  
  
              super().__init__()
  
              _lowercase					:				Any           =			nn.Linear(UpperCamelCase	,      UpperCamelCase	,      bias=UpperCamelCase		)
  
              _lowercase					:				List[Any]           =			nn.Embedding(UpperCamelCase	,      UpperCamelCase		)
              _lowercase					:				Any           =			False
  
              _lowercase					:				Tuple           =			nn.Dropout(p=UpperCamelCase		)
  
              _lowercase					:				Any           =			TaConfig(
                  d_model=UpperCamelCase	,      num_heads=UpperCamelCase	,      d_kv=UpperCamelCase	,      d_ff=UpperCamelCase	,      feed_forward_proj=UpperCamelCase	,      dropout_rate=UpperCamelCase	,      is_decoder=UpperCamelCase	,      is_encoder_decoder=UpperCamelCase	,      )
              _lowercase					:				Any           =			nn.ModuleList()
              for lyr_num in range(UpperCamelCase		):
                     _lowercase					:				Tuple           =			TaBlock(UpperCamelCase		)
                     self.encoders.append(UpperCamelCase		)
  
              _lowercase					:				List[Any]           =			TaLayerNorm(UpperCamelCase		)
              _lowercase					:				Optional[Any]           =			nn.Dropout(p=UpperCamelCase		)
  
  
  
  
       def   lowerCAmelCase_   ( self       :       int	,      UpperCamelCase       :       Optional[int]	,      UpperCamelCase       :       List[Any]		):
  
  
  
  
  
              """simple docstring"""
  
  
  
              _lowercase					:				Optional[int]           =			self.input_proj(UpperCamelCase		)
  
              # terminal relative positional encodings
              _lowercase					:				Optional[int]           =			encoder_inputs.shape[1]
              _lowercase					:				Optional[int]           =			torch.arange(UpperCamelCase	,      device=encoder_inputs.device		)
  
              _lowercase					:				Any           =			encoder_inputs_mask.sum(-1		)
              _lowercase					:				int           =			torch.roll(input_positions.unsqueeze(0		)	,      tuple(seq_lens.tolist()		)	,      dims=0		)
              x += self.position_encoding(UpperCamelCase		)
  
              _lowercase					:				Optional[int]           =			self.dropout_pre(UpperCamelCase		)
  
              # inverted the attention mask
              _lowercase					:				Optional[int]           =			encoder_inputs.size()
              _lowercase					:				List[Any]           =			self.get_extended_attention_mask(UpperCamelCase	,      UpperCamelCase		)
  
              for lyr in self.encoders:
                     _lowercase					:				Optional[int]           =			lyr(UpperCamelCase	,      UpperCamelCase		)[0]
              _lowercase					:				List[str]           =			self.layer_norm(UpperCamelCase		)
  
              return self.dropout_post(UpperCamelCase		), encoder_inputs_mask