code: 
  
  
  import copy
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  _SCREAMING_SNAKE_CASE    :				Dict       =					logging.get_logger(__name__)
  
  _SCREAMING_SNAKE_CASE    :				Tuple       =					{}
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      __magic_name__    						=	'fsmt_decoder'
      def __init__( self    ,      __snake_case=0    ,      __snake_case=0					):
           super().__init__()
           snake_case					=       vocab_size
           snake_case					=       bos_token_id
  
  
  
  
  
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      __magic_name__    						=	'fsmt'
      __magic_name__    						=	{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}
      def __init__( self    ,      __snake_case=["en", "de"]    ,      __snake_case=4_2_0_2_4    ,      __snake_case=4_2_0_2_4    ,      __snake_case="relu"    ,      __snake_case=1_0_2_4    ,      __snake_case=2_0_0    ,      __snake_case=1_0_2_4    ,      __snake_case=4_0_9_6    ,      __snake_case=1_2    ,      __snake_case=1_6    ,      __snake_case=0.0    ,      __snake_case=4_0_9_6    ,      __snake_case=1_2    ,      __snake_case=1_6    ,      __snake_case=0.0    ,      __snake_case=0.0    ,      __snake_case=0.1    ,      __snake_case=0.0    ,      __snake_case=0.02    ,      __snake_case=2    ,      __snake_case=True    ,      __snake_case=True    ,      __snake_case=False    ,      __snake_case=5    ,      __snake_case=1.0    ,      __snake_case=False    ,      __snake_case=True    ,      __snake_case=1    ,      __snake_case=0    ,      __snake_case=2    ,      __snake_case=2    ,      **__snake_case    ,      ):
           snake_case					=       langs
           snake_case					=       src_vocab_size
           snake_case					=       tgt_vocab_size
           snake_case					=       d_model  # encoder_embed_dim and decoder_embed_dim
  
           snake_case					=       encoder_ffn_dim
           snake_case					=       snake_case					=       encoder_layers
           snake_case					=       encoder_attention_heads
           snake_case					=       encoder_layerdrop
           snake_case					=       decoder_layerdrop
           snake_case					=       decoder_ffn_dim
           snake_case					=       decoder_layers
           snake_case					=       decoder_attention_heads
           snake_case					=       max_position_embeddings
           snake_case					=       init_std  # Normal(0, this parameter)
           snake_case					=       activation_function
  
           snake_case					=       DecoderConfig(vocab_size=__snake_case    ,      bos_token_id=__snake_case					)
           if "decoder" in common_kwargs:
                del common_kwargs["decoder"]
  
           snake_case					=       scale_embedding  # scale factor will be sqrt(d_model) if True
  
           # 3 Types of Dropout
           snake_case					=       attention_dropout
           snake_case					=       activation_dropout
           snake_case					=       dropout
  
           snake_case					=       use_cache
           super().__init__(
               pad_token_id=__snake_case    ,      bos_token_id=__snake_case    ,      eos_token_id=__snake_case    ,      decoder_start_token_id=__snake_case    ,      is_encoder_decoder=__snake_case    ,      tie_word_embeddings=__snake_case    ,      forced_eos_token_id=__snake_case    ,      max_length=__snake_case    ,      num_beams=__snake_case    ,      length_penalty=__snake_case    ,      early_stopping=__snake_case    ,      **__snake_case    ,      )
  
  
  
  
      def 	a_    ( self					):
           snake_case					=       copy.deepcopy(self.__dict__					)
           snake_case					=       self.decoder.to_dict()
           snake_case					=       self.__class__.model_type
           return output
  
  
  
  
style_context: 
  
  
  import torch
  
  from accelerate import Accelerator, DistributedType
  
  class 						A__      :
  
  
  
  
  
  
  
      """simple docstring"""
  
      def __enter__( self					):
           if self.enabled:
                snake_case					=       self.model.no_sync()
                self.model_sync_obj.__enter__()
  
           return self
      def __exit__( self    ,      __snake_case    ,      __snake_case    ,      __snake_case					):
           if self.enabled:
                # Average all models on exit
                self._sync_and_avg_model_params()
                self.model_sync_obj.__exit__(__snake_case    ,      __snake_case    ,      __snake_case					)
      def __init__( self    ,      __snake_case    ,      __snake_case    ,      __snake_case    ,      __snake_case = True					):
           if accelerator.distributed_type not in [
               DistributedType.NO,
               DistributedType.MULTI_CPU,
               DistributedType.MULTI_GPU,
           ]:
                raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''					)
           snake_case					=       enabled and accelerator.distributed_type != DistributedType.NO
           snake_case					=       0
           if self.enabled:
                snake_case					=       accelerator
                snake_case					=       model
                snake_case					=       local_sgd_steps
      def 	a_    ( self					):
           self.num_steps += 1
           if not self.enabled:
                return
  
           if self.num_steps % self.local_sgd_steps == 0:
                self._sync_and_avg_model_params()
  
  
  
  
      def 	a_    ( self					):
           self.accelerator.wait_for_everyone()
           with self.accelerator.autocast():
                for param in self.model.parameters():
                     snake_case					=       self.accelerator.reduce(param.data    ,      reduction='''mean'''					)
  
  
  
  
  