
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


snake_case__  							=							logging.getLogger(__name__)

require_version("""pytorch_lightning>=1.0.4""")

snake_case__  							=							{
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
snake_case__  							=							{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
snake_case__  							=							sorted(arg_to_scheduler.keys())
snake_case__  							=							"{" + ", ".join(arg_to_scheduler_choices) + "}"







class 							UpperCAmelCase	(   pl.LightningModule					):

					def __init__(			self      :					Union[str, Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Dict=None ,      lowerCAmelCase      :					int="base" ,      lowerCAmelCase      :					str=None ,      lowerCAmelCase      :					List[str]=None ,      lowerCAmelCase      :					int=None ,      **lowerCAmelCase      :					int ,      ):
											super().__init__()
											# TODO: move to self.save_hyperparameters()
											# self.save_hyperparameters()
											# can also expand arguments into trainer signature for easier reading

											self.save_hyperparameters(__snake_case					)
											lowercase       :    List[str]											=  0
											lowercase       :    List[str]											=  Path(self.hparams.output_dir					)
											lowercase       :    Union[str, Any]											=  self.hparams.cache_dir if self.hparams.cache_dir else None
											if config is None:
																	lowercase       :    Optional[int]											=  AutoConfig.from_pretrained(
																	    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path ,      **({'''num_labels''': num_labels} if num_labels is not None else {}) ,      cache_dir=__snake_case ,      **__snake_case ,      )
											else:
																	lowercase       :    Dict											=  config

											lowercase       :    Any											=  ('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
											for p in extra_model_params:
																	if getattr(self.hparams ,      __snake_case ,      __snake_case					):
																							assert hasattr(self.config ,      __snake_case					), f'''model config doesn\'t have a `{p}` attribute'''
																							setattr(self.config ,      __snake_case ,      getattr(self.hparams ,      __snake_case					)					)

											if tokenizer is None:
																	lowercase       :    List[str]											=  AutoTokenizer.from_pretrained(
																	    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path ,      cache_dir=__snake_case ,      )
											else:
																	lowercase       :    Tuple											=  tokenizer
											lowercase       :    Any											=  MODEL_MODES[mode]
											if model is None:
																	lowercase       :    Optional[Any]											=  self.model_type.from_pretrained(
																	    self.hparams.model_name_or_path ,      from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path					) ,      config=self.config ,      cache_dir=__snake_case ,      )
											else:
																	lowercase       :    Any											=  model

					def        _lowerCAmelCase	(			self      :					Tuple ,      *lowerCAmelCase      :					Any ,      **lowerCAmelCase      :					List[str]					):
											lowercase       :    List[Any]											=  self.model_type.from_pretrained(*__snake_case ,      **__snake_case					)

					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											lowercase       :    str											=  arg_to_scheduler[self.hparams.lr_scheduler]
											lowercase       :    Union[str, Any]											=  get_schedule_func(
											    self.opt ,      num_warmup_steps=self.hparams.warmup_steps ,      num_training_steps=self.total_steps()					)
											lowercase       :    str											=  {'''scheduler''': scheduler, '''interval''': '''step''', '''frequency''': 1}
											return scheduler

					def        _lowerCAmelCase	(			self      :					List[Any]					):
											lowercase       :    Union[str, Any]											=  self.model
											lowercase       :    List[Any]											=  ['''bias''', '''LayerNorm.weight''']
											lowercase       :    List[Any]											=  [
											    {
											        '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay					)],
											        '''weight_decay''': self.hparams.weight_decay,
											    },
											    {
											        '''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay					)],
											        '''weight_decay''': 0.0,
											    },
											]
											if self.hparams.adafactor:
																	lowercase       :    Dict											=  Adafactor(
																	    __snake_case ,      lr=self.hparams.learning_rate ,      scale_parameter=__snake_case ,      relative_step=__snake_case					)

											else:
																	lowercase       :    Optional[Any]											=  AdamW(
																	    __snake_case ,      lr=self.hparams.learning_rate ,      eps=self.hparams.adam_epsilon					)
											lowercase       :    Dict											=  optimizer

											lowercase       :    Optional[int]											=  self.get_lr_scheduler()

											return [optimizer], [scheduler]

					def        _lowerCAmelCase	(			self      :					Dict ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Dict					):
											return self.validation_step(__snake_case ,      __snake_case					)

					def        _lowerCAmelCase	(			self      :					Optional[int] ,      lowerCAmelCase      :					int					):
											return self.validation_end(__snake_case					)

					def        _lowerCAmelCase	(			self      :					Union[str, Any]					):
											lowercase       :    List[Any]											=  max(1 ,      self.hparams.gpus					)  # TODO: consider num_tpu_cores
											lowercase       :    str											=  self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
											return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs

					def        _lowerCAmelCase	(			self      :					List[Any] ,      lowerCAmelCase      :					str					):
											if stage == "test":
																	lowercase       :    str											=  len(self.test_dataloader().dataset					)
											else:
																	lowercase       :    Tuple											=  self.get_dataloader('''train''' ,      self.hparams.train_batch_size ,      shuffle=__snake_case					)
																	lowercase       :    Tuple											=  len(self.train_dataloader().dataset					)

					def        _lowerCAmelCase	(			self      :					Tuple ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Union[str, Any] = False					):
											raise NotImplementedError('''You must implement this for your task'''					)

					def        _lowerCAmelCase	(			self      :					Optional[Any]					):
											return self.train_loader

					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											return self.get_dataloader('''dev''' ,      self.hparams.eval_batch_size ,      shuffle=__snake_case					)

					def        _lowerCAmelCase	(			self      :					Union[str, Any]					):
											return self.get_dataloader('''test''' ,      self.hparams.eval_batch_size ,      shuffle=__snake_case					)

					def        _lowerCAmelCase	(			self      :					Optional[int] ,      lowerCAmelCase      :					Optional[Any]					):
											return os.path.join(
											    self.hparams.data_dir ,      '''cached_{}_{}_{}'''.format(
											        __snake_case ,      list(filter(__snake_case ,      self.hparams.model_name_or_path.split('''/'''					)					)					).pop() ,      str(self.hparams.max_seq_length					) ,      ) ,      )

					@pl.utilities.rank_zero_only
					def        _lowerCAmelCase	(			self      :					Union[str, Any] ,      lowerCAmelCase      :					List[str]					):
											lowercase       :    Optional[int]											=  self.output_dir.joinpath('''best_tfmr'''					)
											lowercase       :    Any											=  self.step_count
											self.model.save_pretrained(__snake_case					)
											self.tokenizer.save_pretrained(__snake_case					)




					@staticmethod
					def        _lowerCAmelCase	(			lowerCAmelCase      :					int ,      lowerCAmelCase      :					int					):
											parser.add_argument(
											    '''--model_name_or_path''' ,      default=__snake_case ,      type=__snake_case ,      required=__snake_case ,      help='''Path to pretrained model or model identifier from huggingface.co/models''' ,      )
											parser.add_argument(
											    '''--config_name''' ,      default='''''' ,      type=__snake_case ,      help='''Pretrained config name or path if not the same as model_name'''					)
											parser.add_argument(
											    '''--tokenizer_name''' ,      default=__snake_case ,      type=__snake_case ,      help='''Pretrained tokenizer name or path if not the same as model_name''' ,      )
											parser.add_argument(
											    '''--cache_dir''' ,      default='''''' ,      type=__snake_case ,      help='''Where do you want to store the pre-trained models downloaded from huggingface.co''' ,      )
											parser.add_argument(
											    '''--encoder_layerdrop''' ,      type=__snake_case ,      help='''Encoder layer dropout probability (Optional). Goes into model.config''' ,      )
											parser.add_argument(
											    '''--decoder_layerdrop''' ,      type=__snake_case ,      help='''Decoder layer dropout probability (Optional). Goes into model.config''' ,      )
											parser.add_argument(
											    '''--dropout''' ,      type=__snake_case ,      help='''Dropout probability (Optional). Goes into model.config''' ,      )
											parser.add_argument(
											    '''--attention_dropout''' ,      type=__snake_case ,      help='''Attention dropout probability (Optional). Goes into model.config''' ,      )
											parser.add_argument('''--learning_rate''' ,      default=5E-5 ,      type=__snake_case ,      help='''The initial learning rate for Adam.'''					)
											parser.add_argument(
											    '''--lr_scheduler''' ,      default='''linear''' ,      choices=__snake_case ,      metavar=__snake_case ,      type=__snake_case ,      help='''Learning rate scheduler''' ,      )
											parser.add_argument('''--weight_decay''' ,      default=0.0 ,      type=__snake_case ,      help='''Weight decay if we apply some.'''					)
											parser.add_argument('''--adam_epsilon''' ,      default=1E-8 ,      type=__snake_case ,      help='''Epsilon for Adam optimizer.'''					)
											parser.add_argument('''--warmup_steps''' ,      default=0 ,      type=__snake_case ,      help='''Linear warmup over warmup_steps.'''					)
											parser.add_argument('''--num_workers''' ,      default=4 ,      type=__snake_case ,      help='''kwarg passed to DataLoader'''					)
											parser.add_argument('''--num_train_epochs''' ,      dest='''max_epochs''' ,      default=3 ,      type=__snake_case					)
											parser.add_argument('''--train_batch_size''' ,      default=32 ,      type=__snake_case					)
											parser.add_argument('''--eval_batch_size''' ,      default=32 ,      type=__snake_case					)
											parser.add_argument('''--adafactor''' ,      action='''store_true'''					)







class 							UpperCAmelCase	(   pl.Callback					):

					def        _lowerCAmelCase	(			self      :					Any ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Optional[Any]					):
											if (
											    trainer.is_global_zero and trainer.global_rank == 0
											):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
																	pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.







class 							UpperCAmelCase	(   pl.Callback					):

					def        _lowerCAmelCase	(			self      :					List[str] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					str					):
											lowercase       :    Any											=  trainer.lr_schedulers[0]['''scheduler''']
											lowercase       :    Any											=  {f'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()					)}
											pl_module.logger.log_metrics(__snake_case					)

					def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					Dict					):
											rank_zero_info('''***** Validation results *****'''					)
											lowercase       :    Optional[int]											=  trainer.callback_metrics
											# Log results
											for key in sorted(__snake_case					):
																	if key not in ["log", "progress_bar"]:
																							rank_zero_info('''{} = {}\n'''.format(__snake_case ,      str(metrics[key]					)					)					)




					def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					str					):
											rank_zero_info('''***** Test results *****'''					)
											lowercase       :    List[Any]											=  trainer.callback_metrics
											# Log and save results to file
											lowercase       :    str											=  os.path.join(pl_module.hparams.output_dir ,      '''test_results.txt'''					)
											with open(__snake_case ,      '''w'''					) as writer:
																	for key in sorted(__snake_case					):
																							if key not in ["log", "progress_bar"]:
																													rank_zero_info('''{} = {}\n'''.format(__snake_case ,      str(metrics[key]					)					)					)
																													writer.write('''{} = {}\n'''.format(__snake_case ,      str(metrics[key]					)					)					)





def        lowerCamelCase_      (			UpperCAmelCase_							:				List[str]							,			UpperCAmelCase_							:				Optional[int]       ):
						parser.add_argument(
						    '''--output_dir'''							,			default=UpperCamelCase_							,			type=UpperCamelCase_							,			required=UpperCamelCase_							,			help='''The output directory where the model predictions and checkpoints will be written.'''							,			)
						parser.add_argument(
						    '''--fp16'''							,			action='''store_true'''							,			help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'''							,			)

						parser.add_argument(
						    '''--fp16_opt_level'''							,			type=UpperCamelCase_							,			default='''O2'''							,			help=(
						        '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
						        '''See details at https://nvidia.github.io/apex/amp.html'''
						    )							,			)
						parser.add_argument('''--n_tpu_cores'''							,			dest='''tpu_cores'''							,			type=UpperCamelCase_       )
						parser.add_argument('''--max_grad_norm'''							,			dest='''gradient_clip_val'''							,			default=1.0							,			type=UpperCamelCase_							,			help='''Max gradient norm'''       )
						parser.add_argument('''--do_train'''							,			action='''store_true'''							,			help='''Whether to run training.'''       )
						parser.add_argument('''--do_predict'''							,			action='''store_true'''							,			help='''Whether to run predictions on the test set.'''       )
						parser.add_argument(
						    '''--gradient_accumulation_steps'''							,			dest='''accumulate_grad_batches'''							,			type=UpperCamelCase_							,			default=1							,			help='''Number of updates steps to accumulate before performing a backward/update pass.'''							,			)
						parser.add_argument('''--seed'''							,			type=UpperCamelCase_							,			default=42							,			help='''random seed for initialization'''       )
						parser.add_argument(
						    '''--data_dir'''							,			default=UpperCamelCase_							,			type=UpperCamelCase_							,			required=UpperCamelCase_							,			help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.'''							,			)




def        lowerCamelCase_      (			UpperCAmelCase_							:				str							,			UpperCAmelCase_							:				Any							,			UpperCAmelCase_							:				Dict=None							,			UpperCAmelCase_							:				int=True							,			UpperCAmelCase_							:				Union[str, Any]=None							,			UpperCAmelCase_							:				Tuple=[]							,			UpperCAmelCase_							:				Tuple=None							,			UpperCAmelCase_							:				Dict=None							,			**UpperCAmelCase_							:				str							,			):
						pl.seed_everything(args.seed       )

						# init model
						lowercase       :    List[str]											=  Path(model.hparams.output_dir       )
						odir.mkdir(exist_ok=UpperCamelCase_       )

						# add custom checkpoints
						if checkpoint_callback is None:
												lowercase       :    int											=  pl.callbacks.ModelCheckpoint(
												    filepath=args.output_dir							,			prefix='''checkpoint'''							,			monitor='''val_loss'''							,			mode='''min'''							,			save_top_k=1       )
						if early_stopping_callback:
												extra_callbacks.append(UpperCamelCase_       )
						if logging_callback is None:
												lowercase       :    Optional[Any]											=  LoggingCallback()

						lowercase       :    Union[str, Any]											=  {}

						# TODO: remove with PyTorch 1.6 since pl uses native amp
						if args.fpaa:
												lowercase       :    Optional[int]											=  16
												# train_params["amp_level"] = args.fp16_opt_level

						if args.gpus > 1:
												lowercase       :    Dict											=  '''auto'''  # "ddp"
												lowercase       :    Any											=  '''ddp'''

						lowercase       :    Optional[int]											=  args.accumulate_grad_batches
						lowercase       :    Tuple											=  None  # extra_train_kwargs.get("profiler", None) #get unwanted logs
						lowercase       :    List[Any]											=  '''auto'''

						lowercase       :    List[Any]											=  pl.Trainer.from_argparse_args(
						    UpperCamelCase_							,			weights_summary=UpperCamelCase_							,			callbacks=[logging_callback] + extra_callbacks + [checkpoint_callback] + [InitCallback()]							,			logger=UpperCamelCase_							,			**UpperCamelCase_							,			)

						if args.do_train:
												trainer.fit(UpperCamelCase_       )

						return trainer




snake_case__  							=							"""
# Installation de Transformers
! pip install transformers datasets
# Pour installer à partir du code source au lieu de la dernière version, commentez la commande ci-dessus et décommentez la suivante.
# ! pip install git+https://github.com/huggingface/transformers.git
"""

snake_case__  							=							[{"""type""": """code""", """content""": INSTALL_CONTENT}]
snake_case__  							=							{
    """{processor_class}""": """FakeProcessorClass""",
    """{model_class}""": """FakeModelClass""",
    """{object_class}""": """FakeObjectClass""",
}


