





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__UpperCAmelCase			:  Tuple										=				logging.get_logger(__name__)

__UpperCAmelCase			:  List[str]										=				{
    "mnaylor/mega-base-wikitext": "https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json",
}







class        __snake_case			(				__lowerCamelCase     ):





       '''simple docstring'''







       lowerCAmelCase__ 						=	"""mega"""




       def __init__(    self					:       List[str]				,							A					:       List[str]=30_522				,							A					:       List[str]=128				,							A					:       int=4				,							A					:       Tuple=256				,							A					:       List[str]=16				,							A					:       Dict=True				,							A					:       List[Any]=64				,							A					:       str=False				,							A					:       Optional[int]=-1				,							A					:       str=None				,							A					:       str=True				,							A					:       List[str]="scalenorm"				,							A					:       Tuple=True				,							A					:       List[Any]="silu"				,							A					:       Dict="softmax"				,							A					:       Optional[int]=0.1				,							A					:       Optional[Any]=0.1				,							A					:       Any=0.1				,							A					:       str=False				,							A					:       Union[str, Any]=True				,							A					:       List[Any]=256				,							A					:       Dict=True				,							A					:       Union[str, Any]=0.1				,							A					:       Union[str, Any]=2_048				,							A					:       List[str]=False				,							A					:       Dict=2				,							A					:       Union[str, Any]=0.02				,							A					:       Dict=0.2				,							A					:       Dict=0.02				,							A					:       int=1.0				,							A					:       Union[str, Any]=1				,							A					:       Optional[Any]=0				,							A					:       Union[str, Any]=2				,							A					:       int="rotary"				,							A					:       str=None				,							A					:       List[Any]=True				,							A					:       Optional[int]=True				,							**A					:       Union[str, Any]				,							):
         super().__init__(pad_token_id=_lowerCAmelCase				,							bos_token_id=_lowerCAmelCase				,							eos_token_id=_lowerCAmelCase				,							**_lowerCAmelCase						)

         __snake_case:		Optional[int]					    =		vocab_size
         __snake_case:		Union[str, Any]					    =		hidden_size
         __snake_case:		Tuple					    =		num_hidden_layers
         __snake_case:		int					    =		activation
         __snake_case:		List[Any]					    =		attention_activation
         __snake_case:		List[Any]					    =		intermediate_size
         __snake_case:		Dict					    =		ema_projection_size
         __snake_case:		List[Any]					    =		bidirectional
         __snake_case:		int					    =		shared_representation_size
         __snake_case:		Tuple					    =		use_chunking
         __snake_case:		List[Any]					    =		chunk_size
         __snake_case:		List[str]					    =		truncation
         __snake_case:		Union[str, Any]					    =		normalize_before_mega
         __snake_case:		Dict					    =		normalization_type
         __snake_case:		Dict					    =		norm_affine
         __snake_case:		List[Any]					    =		dropout_prob
         __snake_case:		Tuple					    =		hidden_dropout_prob
         __snake_case:		Union[str, Any]					    =		attention_probs_dropout_prob
         __snake_case:		Union[str, Any]					    =		use_feature_dropout
         __snake_case:		Union[str, Any]					    =		use_normalized_ffn
         __snake_case:		List[Any]					    =		nffn_hidden_size
         __snake_case:		Union[str, Any]					    =		normalize_before_ffn
         __snake_case:		Any					    =		nffn_activation_dropout_prob
         __snake_case:		List[Any]					    =		max_positions
         __snake_case:		Dict					    =		add_token_type_embeddings
         __snake_case:		int					    =		type_vocab_size
         __snake_case:		Any					    =		initializer_range
         __snake_case:		int					    =		ema_delta_alpha_range
         __snake_case:		Union[str, Any]					    =		ema_beta_range
         __snake_case:		Tuple					    =		ema_gamma_omega_range
         __snake_case:		List[str]					    =		relative_positional_bias
         __snake_case:		Any					    =		use_cache
         __snake_case:		Union[str, Any]					    =		classifier_dropout
         __snake_case:		Union[str, Any]					    =		add_lm_hidden_dense_layer
         __snake_case:		int					    =		1  # not used but required by Hugging Face







class        __snake_case			(				__lowerCamelCase     ):





       '''simple docstring'''







       @property
       def 				UpperCAmelCase__					(    self					:       Tuple						):
         if self.task == "multiple-choice":
           __snake_case:		Any					    =		{0: "batch", 1: "choice", 2: "sequence"}
         else:
           __snake_case:		Any					    =		{0: "batch", 1: "sequence"}
         return OrderedDict(
             [
                 ("""input_ids""", dynamic_axis),
                 ("""attention_mask""", dynamic_axis),
             ]						)












import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only

from utils import save_json



def    UpperCAmelCase__	(  lowerCamelCase ):
  lowercase     :Optional[Any]   						=    filter(lambda lowerCamelCase : p.requires_grad,      model.parameters() )
  lowercase     :Optional[int]   						=    sum([np.prod(p.size() ) for p in model_parameters] )
  return params


_UpperCAmelCase						:    Optional[Any]      =  logging.getLogger(__name__)




class  __lowerCAmelCase							(				pl.Callback):






       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]							,       _lowerCAmelCase:       List[Any]							,       _lowerCAmelCase:       Union[str, Any]					):
         lowercase     :Dict   						=    {F"lr_group_{i}": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups					)}
         pl_module.logger.log_metrics(_lowerCAmelCase					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       Optional[Any]							,       _lowerCAmelCase:       pl.Trainer							,       _lowerCAmelCase:       pl.LightningModule							,       _lowerCAmelCase:       str							,       _lowerCAmelCase:       List[str]=True					):
         logger.info(F"***** {type_path} results at step {trainer.global_step:05d} *****"					)
         lowercase     :Tuple   						=    trainer.callback_metrics
         trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]}					)
         # Log results
         lowercase     :Optional[int]   						=    Path(pl_module.hparams.output_dir					)
         if type_path == "test":
           lowercase     :List[Any]   						=    od / "test_results.txt"
           lowercase     :Optional[int]   						=    od / "test_generations.txt"
         else:
           # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
           # If people want this it will be easy enough to add back.
           lowercase     :int   						=    od / F"{type_path}_results/{trainer.global_step:05d}.txt"
           lowercase     :Tuple   						=    od / F"{type_path}_generations/{trainer.global_step:05d}.txt"
           results_file.parent.mkdir(exist_ok=_lowerCAmelCase					)
           generations_file.parent.mkdir(exist_ok=_lowerCAmelCase					)
         with open(_lowerCAmelCase							,       "a+"					) as writer:
           for key in sorted(_lowerCAmelCase					):
             if key in ["log", "progress_bar", "preds"]:
               continue
             lowercase     :int   						=    metrics[key]
             if isinstance(_lowerCAmelCase							,       torch.Tensor					):
               lowercase     :int   						=    val.item()
             lowercase     :Optional[Any]   						=    F"{key}: {val:.6f}\n"
             writer.write(_lowerCAmelCase					)

         if not save_generations:
           return

         if "preds" in metrics:
           lowercase     :str   						=    "\n".join(metrics["preds"]					)
           generations_file.open("w+"					).write(_lowerCAmelCase					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       Optional[int]							,       _lowerCAmelCase:       Tuple							,       _lowerCAmelCase:       Tuple					):
         try:
           lowercase     :Optional[int]   						=    pl_module.model.model.num_parameters()
         except AttributeError:
           lowercase     :Dict   						=    pl_module.model.num_parameters()

         lowercase     :Any   						=    count_trainable_parameters(_lowerCAmelCase					)
         # mp stands for million parameters
         trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1e6, "grad_mp": n_trainable_pars / 1e6}					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       Union[str, Any]							,       _lowerCAmelCase:       pl.Trainer							,       _lowerCAmelCase:       pl.LightningModule					):
         save_json(pl_module.metrics							,       pl_module.metrics_save_path					)
         return self._write_logs(_lowerCAmelCase							,       _lowerCAmelCase							,       "test"					)






       @rank_zero_only
       def        SCREAMING_SNAKE_CASE     ( self:       List[str]							,       _lowerCAmelCase:       pl.Trainer							,       _lowerCAmelCase:       Dict					):
         save_json(pl_module.metrics							,       pl_module.metrics_save_path					)
         # Uncommenting this will save val generations
         # return self._write_logs(trainer, pl_module, "valid")







def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase,      lowerCamelCase=1,      lowerCamelCase=False ):
  if metric == "rouge2":
    lowercase     :str   						=    "{val_avg_rouge2:.4f}-{step_count}"
  elif metric == "bleu":
    lowercase     :int   						=    "{val_avg_bleu:.4f}-{step_count}"
  elif metric == "loss":
    lowercase     :List[str]   						=    "{val_avg_loss:.4f}-{step_count}"
  else:
    raise NotImplementedError(
        F"seq2seq callbacks only support rouge2, bleu and loss, got {metric}, You can make your own by adding to"
        " this function." )

  lowercase     :int   						=    ModelCheckpoint(
      dirpath=lowerCamelCase,      filename=lowerCamelCase,      monitor=F"val_{metric}",      mode="min" if "loss" in metric else "max",      save_top_k=lowerCamelCase,      )
  return checkpoint_callback







def    UpperCAmelCase__	(  lowerCamelCase,      lowerCamelCase ):
  return EarlyStopping(
      monitor=F"val_{metric}",      mode="min" if "loss" in metric else "max",      patience=lowerCamelCase,      verbose=lowerCamelCase,      )



