





'''simple docstring'''







import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


snake_case_  :							List[str]            =	logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

snake_case_  :							List[str]            =	{
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
snake_case_  :							int            =	{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
snake_case_  :							Any            =	sorted(arg_to_scheduler.keys())
snake_case_  :							Union[str, Any]            =	"{" + ", ".join(arg_to_scheduler_choices) + "}"


class        __a   (pl.LightningModule  ):

							def __init__(				self				:  str							,  __magic_name__				:  argparse.Namespace							,  __magic_name__				:  Tuple=None							,  __magic_name__				:  Any="base"							,  __magic_name__				:  str=None							,  __magic_name__				:  Optional[int]=None							,  __magic_name__				:  Any=None							,  **__magic_name__				:  str							,  )       ->							Optional[int]:







													"""simple docstring"""
													super().__init__()
													# TODO: move to self.save_hyperparameters()
													# self.save_hyperparameters()
													# can also expand arguments into trainer signature for easier reading

													self.save_hyperparameters(__magic_name__ )
													UpperCAmelCase_      :   Any      =	0
													UpperCAmelCase_      :   Union[str, Any]      =	Path(self.hparams.output_dir )
													UpperCAmelCase_      :   List[Any]      =	self.hparams.cache_dir if self.hparams.cache_dir else None
													if config is None:
																			UpperCAmelCase_      :   Optional[Any]      =	AutoConfig.from_pretrained(
																			    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path							,  **({'''num_labels''': num_labels} if num_labels is not None else {})							,  cache_dir=__magic_name__							,  **__magic_name__							,  )
													else:
																			UpperCAmelCase_      :   PretrainedConfig      =	config

													UpperCAmelCase_      :   Tuple      =	('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
													for p in extra_model_params:
																			if getattr(self.hparams							,  __magic_name__							,  __magic_name__ ):
																									assert hasattr(self.config							,  __magic_name__ ), F"""model config doesn't have a `{p}` attribute"""
																									setattr(self.config							,  __magic_name__							,  getattr(self.hparams							,  __magic_name__ ) )

													if tokenizer is None:
																			UpperCAmelCase_      :   Any      =	AutoTokenizer.from_pretrained(
																			    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path							,  cache_dir=__magic_name__							,  )
													else:
																			UpperCAmelCase_      :   PreTrainedTokenizer      =	tokenizer
													UpperCAmelCase_      :   str      =	MODEL_MODES[mode]
													if model is None:
																			UpperCAmelCase_      :   Dict      =	self.model_type.from_pretrained(
																			    self.hparams.model_name_or_path							,  from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path )							,  config=self.config							,  cache_dir=__magic_name__							,  )
													else:
																			UpperCAmelCase_      :   Optional[Any]      =	model

							def      UpperCAmelCase__						(				self				:  int							,  *__magic_name__				:  Any							,  **__magic_name__				:  Dict )       ->							Any:







													"""simple docstring"""
													UpperCAmelCase_      :   str      =	self.model_type.from_pretrained(*__magic_name__							,  **__magic_name__ )

							def      UpperCAmelCase__						(				self				:  Any )       ->							List[Any]:







													"""simple docstring"""
													UpperCAmelCase_      :   Union[str, Any]      =	arg_to_scheduler[self.hparams.lr_scheduler]
													UpperCAmelCase_      :   Union[str, Any]      =	get_schedule_func(
													    self.opt							,  num_warmup_steps=self.hparams.warmup_steps							,  num_training_steps=self.total_steps() )
													UpperCAmelCase_      :   Optional[int]      =	{'''scheduler''': scheduler, '''interval''': '''step''', '''frequency''': 1}
													return scheduler

							def      UpperCAmelCase__						(				self				:  Optional[int] )       ->							Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_      :   Union[str, Any]      =	self.model
													UpperCAmelCase_      :   int      =	['''bias''', '''LayerNorm.weight''']
													UpperCAmelCase_      :   Optional[Any]      =	[
													    {
													        '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay )],
													        '''weight_decay''': self.hparams.weight_decay,
													    },
													    {
													        '''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay )],
													        '''weight_decay''': 0.0,
													    },
													]
													if self.hparams.adafactor:
																			UpperCAmelCase_      :   Dict      =	Adafactor(
																			    __magic_name__							,  lr=self.hparams.learning_rate							,  scale_parameter=__magic_name__							,  relative_step=__magic_name__ )

													else:
																			UpperCAmelCase_      :   Tuple      =	AdamW(
																			    __magic_name__							,  lr=self.hparams.learning_rate							,  eps=self.hparams.adam_epsilon )
													UpperCAmelCase_      :   Union[str, Any]      =	optimizer

													UpperCAmelCase_      :   str      =	self.get_lr_scheduler()

													return [optimizer], [scheduler]

							def      UpperCAmelCase__						(				self				:  List[str]							,  __magic_name__				:  Tuple							,  __magic_name__				:  Union[str, Any] )       ->							str:







													"""simple docstring"""
													return self.validation_step(__magic_name__							,  __magic_name__ )

							def      UpperCAmelCase__						(				self				:  Dict							,  __magic_name__				:  str )       ->							Optional[Any]:







													"""simple docstring"""
													return self.validation_end(__magic_name__ )

							def      UpperCAmelCase__						(				self				:  Optional[Any] )       ->							int:







													"""simple docstring"""
													UpperCAmelCase_      :   Tuple      =	max(1							,  self.hparams.gpus )  # TODO: consider num_tpu_cores
													UpperCAmelCase_      :   Tuple      =	self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
													return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs

							def      UpperCAmelCase__						(				self				:  Any							,  __magic_name__				:  Union[str, Any] )       ->							Dict:







													"""simple docstring"""
													if stage == "test":
																			UpperCAmelCase_      :   Optional[int]      =	len(self.test_dataloader().dataset )
													else:
																			UpperCAmelCase_      :   Dict      =	self.get_dataloader('''train'''							,  self.hparams.train_batch_size							,  shuffle=__magic_name__ )
																			UpperCAmelCase_      :   List[str]      =	len(self.train_dataloader().dataset )

							def      UpperCAmelCase__						(				self				:  str							,  __magic_name__				:  str							,  __magic_name__				:  int							,  __magic_name__				:  bool = False )       ->							int:







													"""simple docstring"""
													raise NotImplementedError('''You must implement this for your task''' )

							def      UpperCAmelCase__						(				self				:  Dict )       ->							Union[str, Any]:







													"""simple docstring"""
													return self.train_loader

							def      UpperCAmelCase__						(				self				:  Any )       ->							Optional[int]:







													"""simple docstring"""
													return self.get_dataloader('''dev'''							,  self.hparams.eval_batch_size							,  shuffle=__magic_name__ )

							def      UpperCAmelCase__						(				self				:  int )       ->							int:







													"""simple docstring"""
													return self.get_dataloader('''test'''							,  self.hparams.eval_batch_size							,  shuffle=__magic_name__ )

							def      UpperCAmelCase__						(				self				:  str							,  __magic_name__				:  List[str] )       ->							Optional[Any]:







													"""simple docstring"""
													return os.path.join(
													    self.hparams.data_dir							,  '''cached_{}_{}_{}'''.format(
													        __magic_name__							,  list(filter(__magic_name__							,  self.hparams.model_name_or_path.split('''/''' ) ) ).pop()							,  str(self.hparams.max_seq_length )							,  )							,  )

							@pl.utilities.rank_zero_only
							def      UpperCAmelCase__						(				self				:  str							,  __magic_name__				:  Dict[str, Any] )       ->							None:







													"""simple docstring"""
													UpperCAmelCase_      :   int      =	self.output_dir.joinpath('''best_tfmr''' )
													UpperCAmelCase_      :   int      =	self.step_count
													self.model.save_pretrained(__magic_name__ )
													self.tokenizer.save_pretrained(__magic_name__ )



							@staticmethod
							def      UpperCAmelCase__						(				__magic_name__				:  int							,  __magic_name__				:  str )       ->							str:







													"""simple docstring"""
													parser.add_argument(
													    '''--model_name_or_path'''							,  default=__magic_name__							,  type=__magic_name__							,  required=__magic_name__							,  help='''Path to pretrained model or model identifier from huggingface.co/models'''							,  )
													parser.add_argument(
													    '''--config_name'''							,  default=''''''							,  type=__magic_name__							,  help='''Pretrained config name or path if not the same as model_name''' )
													parser.add_argument(
													    '''--tokenizer_name'''							,  default=__magic_name__							,  type=__magic_name__							,  help='''Pretrained tokenizer name or path if not the same as model_name'''							,  )
													parser.add_argument(
													    '''--cache_dir'''							,  default=''''''							,  type=__magic_name__							,  help='''Where do you want to store the pre-trained models downloaded from huggingface.co'''							,  )
													parser.add_argument(
													    '''--encoder_layerdrop'''							,  type=__magic_name__							,  help='''Encoder layer dropout probability (Optional). Goes into model.config'''							,  )
													parser.add_argument(
													    '''--decoder_layerdrop'''							,  type=__magic_name__							,  help='''Decoder layer dropout probability (Optional). Goes into model.config'''							,  )
													parser.add_argument(
													    '''--dropout'''							,  type=__magic_name__							,  help='''Dropout probability (Optional). Goes into model.config'''							,  )
													parser.add_argument(
													    '''--attention_dropout'''							,  type=__magic_name__							,  help='''Attention dropout probability (Optional). Goes into model.config'''							,  )
													parser.add_argument('''--learning_rate'''							,  default=5E-5							,  type=__magic_name__							,  help='''The initial learning rate for Adam.''' )
													parser.add_argument(
													    '''--lr_scheduler'''							,  default='''linear'''							,  choices=__magic_name__							,  metavar=__magic_name__							,  type=__magic_name__							,  help='''Learning rate scheduler'''							,  )
													parser.add_argument('''--weight_decay'''							,  default=0.0							,  type=__magic_name__							,  help='''Weight decay if we apply some.''' )
													parser.add_argument('''--adam_epsilon'''							,  default=1E-8							,  type=__magic_name__							,  help='''Epsilon for Adam optimizer.''' )
													parser.add_argument('''--warmup_steps'''							,  default=0							,  type=__magic_name__							,  help='''Linear warmup over warmup_steps.''' )
													parser.add_argument('''--num_workers'''							,  default=4							,  type=__magic_name__							,  help='''kwarg passed to DataLoader''' )
													parser.add_argument('''--num_train_epochs'''							,  dest='''max_epochs'''							,  default=3							,  type=__magic_name__ )
													parser.add_argument('''--train_batch_size'''							,  default=32							,  type=__magic_name__ )
													parser.add_argument('''--eval_batch_size'''							,  default=32							,  type=__magic_name__ )
													parser.add_argument('''--adafactor'''							,  action='''store_true''' )





class        __a   (pl.Callback  ):

							def      UpperCAmelCase__						(				self				:  List[Any]							,  __magic_name__				:  Optional[Any]							,  __magic_name__				:  List[Any] )       ->							Any:







													"""simple docstring"""
													if (
													    trainer.is_global_zero and trainer.global_rank == 0
													):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
																			pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.





class        __a   (pl.Callback  ):

							def      UpperCAmelCase__						(				self				:  Tuple							,  __magic_name__				:  int							,  __magic_name__				:  Any )       ->							List[str]:







													"""simple docstring"""
													UpperCAmelCase_      :   int      =	trainer.lr_schedulers[0]['''scheduler''']
													UpperCAmelCase_      :   int      =	{F"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr() )}
													pl_module.logger.log_metrics(__magic_name__ )

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  __magic_name__				:  pl.Trainer							,  __magic_name__				:  pl.LightningModule )       ->							Any:







													"""simple docstring"""
													rank_zero_info('''***** Validation results *****''' )
													UpperCAmelCase_      :   List[str]      =	trainer.callback_metrics
													# Log results
													for key in sorted(__magic_name__ ):
																			if key not in ["log", "progress_bar"]:
																									rank_zero_info('''{} = {}\n'''.format(__magic_name__							,  str(metrics[key] ) ) )



							def      UpperCAmelCase__						(				self				:  Dict							,  __magic_name__				:  pl.Trainer							,  __magic_name__				:  pl.LightningModule )       ->							List[Any]:







													"""simple docstring"""
													rank_zero_info('''***** Test results *****''' )
													UpperCAmelCase_      :   int      =	trainer.callback_metrics
													# Log and save results to file
													UpperCAmelCase_      :   Any      =	os.path.join(pl_module.hparams.output_dir							,  '''test_results.txt''' )
													with open(__magic_name__							,  '''w''' ) as writer:
																			for key in sorted(__magic_name__ ):
																									if key not in ["log", "progress_bar"]:
																															rank_zero_info('''{} = {}\n'''.format(__magic_name__							,  str(metrics[key] ) ) )
																															writer.write('''{} = {}\n'''.format(__magic_name__							,  str(metrics[key] ) ) )





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      Union[str, Any],      SCREAMING_SNAKE_CASE__   :      Any )			->						None:
						#  To allow all pl args uncomment the following line
						#  parser = pl.Trainer.add_argparse_args(parser)
						parser.add_argument(
						    '''--output_dir''',      default=SCREAMING_SNAKE_CASE__,      type=SCREAMING_SNAKE_CASE__,      required=SCREAMING_SNAKE_CASE__,      help='''The output directory where the model predictions and checkpoints will be written.''',      )
						parser.add_argument(
						    '''--fp16''',      action='''store_true''',      help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit''',      )

						parser.add_argument(
						    '''--fp16_opt_level''',      type=SCREAMING_SNAKE_CASE__,      default='''O2''',      help=(
						        '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
						        '''See details at https://nvidia.github.io/apex/amp.html'''
						    ),      )
						parser.add_argument('''--n_tpu_cores''',      dest='''tpu_cores''',      type=SCREAMING_SNAKE_CASE__ )
						parser.add_argument('''--max_grad_norm''',      dest='''gradient_clip_val''',      default=1.0,      type=SCREAMING_SNAKE_CASE__,      help='''Max gradient norm''' )
						parser.add_argument('''--do_train''',      action='''store_true''',      help='''Whether to run training.''' )
						parser.add_argument('''--do_predict''',      action='''store_true''',      help='''Whether to run predictions on the test set.''' )
						parser.add_argument(
						    '''--gradient_accumulation_steps''',      dest='''accumulate_grad_batches''',      type=SCREAMING_SNAKE_CASE__,      default=1,      help='''Number of updates steps to accumulate before performing a backward/update pass.''',      )
						parser.add_argument('''--seed''',      type=SCREAMING_SNAKE_CASE__,      default=42,      help='''random seed for initialization''' )
						parser.add_argument(
						    '''--data_dir''',      default=SCREAMING_SNAKE_CASE__,      type=SCREAMING_SNAKE_CASE__,      required=SCREAMING_SNAKE_CASE__,      help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.''',      )





def 							lowerCamelCase_   (	SCREAMING_SNAKE_CASE__   :      BaseTransformer,      SCREAMING_SNAKE_CASE__   :      argparse.Namespace,      SCREAMING_SNAKE_CASE__   :      Union[str, Any]=None,      SCREAMING_SNAKE_CASE__   :      Union[str, Any]=True,      SCREAMING_SNAKE_CASE__   :      List[Any]=None,      SCREAMING_SNAKE_CASE__   :      str=[],      SCREAMING_SNAKE_CASE__   :      List[str]=None,      SCREAMING_SNAKE_CASE__   :      List[str]=None,      **SCREAMING_SNAKE_CASE__   :      Tuple,      )			->						Optional[int]:
						pl.seed_everything(args.seed )

						# init model
						UpperCAmelCase_      :   int      =	Path(model.hparams.output_dir )
						odir.mkdir(exist_ok=SCREAMING_SNAKE_CASE__ )

						# add custom checkpoints
						if checkpoint_callback is None:
												UpperCAmelCase_      :   Union[str, Any]      =	pl.callbacks.ModelCheckpoint(
												    filepath=args.output_dir,      prefix='''checkpoint''',      monitor='''val_loss''',      mode='''min''',      save_top_k=1 )
						if early_stopping_callback:
												extra_callbacks.append(SCREAMING_SNAKE_CASE__ )
						if logging_callback is None:
												UpperCAmelCase_      :   Any      =	LoggingCallback()

						UpperCAmelCase_      :   Dict      =	{}

						# TODO: remove with PyTorch 1.6 since pl uses native amp
						if args.fpaa:
												UpperCAmelCase_      :   int      =	16
												# train_params["amp_level"] = args.fp16_opt_level

						if args.gpus > 1:
												UpperCAmelCase_      :   str      =	'''auto'''  # "ddp"
												UpperCAmelCase_      :   List[str]      =	'''ddp'''

						UpperCAmelCase_      :   Any      =	args.accumulate_grad_batches
						UpperCAmelCase_      :   Tuple      =	None  # extra_train_kwargs.get("profiler", None) #get unwanted logs
						UpperCAmelCase_      :   Union[str, Any]      =	'''auto'''

						UpperCAmelCase_      :   int      =	pl.Trainer.from_argparse_args(
						    SCREAMING_SNAKE_CASE__,      weights_summary=SCREAMING_SNAKE_CASE__,      callbacks=[logging_callback] + extra_callbacks + [checkpoint_callback] + [InitCallback()],      logger=SCREAMING_SNAKE_CASE__,      **SCREAMING_SNAKE_CASE__,      )

						if args.do_train:
												trainer.fit(SCREAMING_SNAKE_CASE__ )

						return trainer








"""simple docstring"""
import argparse

from transformers import TaConfig, TaForConditionalGeneration, load_tf_weights_in_ta
from transformers.utils import logging


logging.set_verbosity_info()







def       UpperCamelCase_					(     lowerCamelCase							:					List[str]     , lowerCamelCase							:					Optional[int]     , lowerCamelCase							:					List[str]   )						->     Optional[int]:



       """simple docstring"""







       __magic_name__			:		int        =     TaConfig.from_json_file(lowerCamelCase   )
       print(f"""Building PyTorch model from configuration: {config}"""   )
       __magic_name__			:		Union[str, Any]        =     TaForConditionalGeneration(lowerCamelCase   )

       # Load weights from tf checkpoint
       load_tf_weights_in_ta(lowerCamelCase     , lowerCamelCase     , lowerCamelCase   )

       # Save pytorch-model
       print(f"""Save PyTorch model to {pytorch_dump_path}"""   )
       model.save_pretrained(lowerCamelCase   )


if __name__ == "__main__":
 A									=						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
 )
 parser.add_argument(
     """--config_file""",
     default=None,
     type=str,
     required=True,
     help=(
         """The config json file corresponding to the pre-trained T5 model. \nThis specifies the model architecture."""
     ),
 )
 parser.add_argument(
     """--pytorch_dump_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
 )
 A									=						parser.parse_args()
 convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)
