

import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


lowerCAmelCase						:   Union[str, Any]										=    logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

lowerCAmelCase						:   Optional[Any]										=    {
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
lowerCAmelCase						:   Any										=    {
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
lowerCAmelCase						:   Any										=    sorted(arg_to_scheduler.keys())
lowerCAmelCase						:   List[str]										=    '''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''
class     SCREAMING_SNAKE_CASE__     (						pl.LightningModule		):





       '''simple docstring'''




       def __init__(						self       :						Union[str, Any]    ,		lowerCAmelCase__       :						argparse.Namespace    ,		lowerCAmelCase__       :						Dict=None    ,		lowerCAmelCase__       :						Tuple="base"    ,		lowerCAmelCase__       :						str=None    ,		lowerCAmelCase__       :						List[Any]=None    ,		lowerCAmelCase__       :						List[str]=None    ,		**lowerCAmelCase__       :						Optional[Any]    ,		)			->    Optional[int]:
             super().__init__()
             # TODO: move to self.save_hyperparameters()
             # self.save_hyperparameters()
             # can also expand arguments into trainer signature for easier reading

             self.save_hyperparameters(UpperCAmelCase_							)
             snake_case__					=   0
             snake_case__					=   Path(self.hparams.output_dir							)
             snake_case__					=   self.hparams.cache_dir if self.hparams.cache_dir else None
             if config is None:
                   snake_case__					=   AutoConfig.from_pretrained(
                       self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path    ,		**({"""num_labels""": num_labels} if num_labels is not None else {})    ,		cache_dir=UpperCAmelCase_    ,		**UpperCAmelCase_    ,		)
             else:
                   snake_case__					=   config

             snake_case__					=   ('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
             for p in extra_model_params:
                   if getattr(self.hparams    ,		UpperCAmelCase_    ,		UpperCAmelCase_							):
                         assert hasattr(self.config    ,		UpperCAmelCase_							), f'''model config doesn\'t have a `{p}` attribute'''
                         setattr(self.config    ,		UpperCAmelCase_    ,		getattr(self.hparams    ,		UpperCAmelCase_							)							)

             if tokenizer is None:
                   snake_case__					=   AutoTokenizer.from_pretrained(
                       self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path    ,		cache_dir=UpperCAmelCase_    ,		)
             else:
                   snake_case__					=   tokenizer
             snake_case__					=   MODEL_MODES[mode]
             if model is None:
                   snake_case__					=   self.model_type.from_pretrained(
                       self.hparams.model_name_or_path    ,		from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path							)    ,		config=self.config    ,		cache_dir=UpperCAmelCase_    ,		)
             else:
                   snake_case__					=   model

       def UpperCAmelCase_						(						self       :						Tuple    ,		*lowerCAmelCase__       :						Optional[Any]    ,		**lowerCAmelCase__       :						Any							)			->    int:
             snake_case__					=   self.model_type.from_pretrained(*UpperCAmelCase_    ,		**UpperCAmelCase_							)

       def UpperCAmelCase_						(						self       :						str							)			->    Tuple:
             snake_case__					=   arg_to_scheduler[self.hparams.lr_scheduler]
             snake_case__					=   get_schedule_func(
                 self.opt    ,		num_warmup_steps=self.hparams.warmup_steps    ,		num_training_steps=self.total_steps()							)
             snake_case__					=   {'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
             return scheduler

       def UpperCAmelCase_						(						self       :						Union[str, Any]							)			->    Any:
             snake_case__					=   self.model
             snake_case__					=   ['bias', 'LayerNorm.weight']
             snake_case__					=   [
                 {
                     'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay							)],
                     'weight_decay': self.hparams.weight_decay,
                 },
                 {
                     'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay							)],
                     'weight_decay': 0.0,
                 },
             ]
             if self.hparams.adafactor:
                   snake_case__					=   Adafactor(
                       UpperCAmelCase_    ,		lr=self.hparams.learning_rate    ,		scale_parameter=UpperCAmelCase_    ,		relative_step=UpperCAmelCase_							)

             else:
                   snake_case__					=   AdamW(
                       UpperCAmelCase_    ,		lr=self.hparams.learning_rate    ,		eps=self.hparams.adam_epsilon							)
             snake_case__					=   optimizer

             snake_case__					=   self.get_lr_scheduler()

             return [optimizer], [scheduler]

       def UpperCAmelCase_						(						self       :						str    ,		lowerCAmelCase__       :						List[Any]    ,		lowerCAmelCase__       :						str							)			->    Tuple:
             return self.validation_step(UpperCAmelCase_    ,		UpperCAmelCase_							)

       def UpperCAmelCase_						(						self       :						Any    ,		lowerCAmelCase__       :						Optional[int]							)			->    Any:
             return self.validation_end(UpperCAmelCase_							)

       def UpperCAmelCase_						(						self       :						Optional[Any]							)			->    Union[str, Any]:
             snake_case__					=   max(1    ,		self.hparams.gpus							)  # TODO: consider num_tpu_cores
             snake_case__					=   self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
             return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs

       def UpperCAmelCase_						(						self       :						Tuple    ,		lowerCAmelCase__       :						Union[str, Any]							)			->    List[str]:
             if mode == "test":
                   snake_case__					=   len(self.test_dataloader().dataset							)
             else:
                   snake_case__					=   self.get_dataloader("""train"""    ,		self.hparams.train_batch_size    ,		shuffle=UpperCAmelCase_							)
                   snake_case__					=   len(self.train_dataloader().dataset							)

       def UpperCAmelCase_						(						self       :						int    ,		lowerCAmelCase__       :						str    ,		lowerCAmelCase__       :						int    ,		lowerCAmelCase__       :						bool = False							)			->    str:
             raise NotImplementedError("""You must implement this for your task"""							)

       def UpperCAmelCase_						(						self       :						Optional[int]							)			->    Any:
             return self.train_loader

       def UpperCAmelCase_						(						self       :						int							)			->    List[Any]:
             return self.get_dataloader("""dev"""    ,		self.hparams.eval_batch_size    ,		shuffle=UpperCAmelCase_							)

       def UpperCAmelCase_						(						self       :						Optional[Any]							)			->    Dict:
             return self.get_dataloader("""test"""    ,		self.hparams.eval_batch_size    ,		shuffle=UpperCAmelCase_							)

       def UpperCAmelCase_						(						self       :						Tuple    ,		lowerCAmelCase__       :						Dict							)			->    Tuple:
             return os.path.join(
                 self.hparams.data_dir    ,		"""cached_{}_{}_{}""".format(
                     UpperCAmelCase_    ,		list(filter(UpperCAmelCase_    ,		self.hparams.model_name_or_path.split("""/"""							)							)							).pop()    ,		str(self.hparams.max_seq_length							)    ,		)    ,		)

       @pl.utilities.rank_zero_only
       def UpperCAmelCase_						(						self       :						Optional[Any]    ,		lowerCAmelCase__       :						Dict[str, Any]							)			->    Any:
             snake_case__					=   self.output_dir.joinpath("""best_tfmr"""							)
             snake_case__					=   self.step_count
             self.model.save_pretrained(UpperCAmelCase_							)
             self.tokenizer.save_pretrained(UpperCAmelCase_							)

       @staticmethod
       def UpperCAmelCase_						(						lowerCAmelCase__       :						Optional[int]    ,		lowerCAmelCase__       :						Any							)			->    List[str]:
             parser.add_argument(
                 """--model_name_or_path"""    ,		default=UpperCAmelCase_    ,		type=UpperCAmelCase_    ,		required=UpperCAmelCase_    ,		help="""Path to pretrained model or model identifier from huggingface.co/models"""    ,		)
             parser.add_argument(
                 """--config_name"""    ,		default=""""""    ,		type=UpperCAmelCase_    ,		help="""Pretrained config name or path if not the same as model_name"""							)
             parser.add_argument(
                 """--tokenizer_name"""    ,		default=UpperCAmelCase_    ,		type=UpperCAmelCase_    ,		help="""Pretrained tokenizer name or path if not the same as model_name"""    ,		)
             parser.add_argument(
                 """--cache_dir"""    ,		default=""""""    ,		type=UpperCAmelCase_    ,		help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""    ,		)
             parser.add_argument(
                 """--encoder_layerdrop"""    ,		type=UpperCAmelCase_    ,		help="""Encoder layer dropout probability (Optional). Goes into model.config"""    ,		)
             parser.add_argument(
                 """--decoder_layerdrop"""    ,		type=UpperCAmelCase_    ,		help="""Decoder layer dropout probability (Optional). Goes into model.config"""    ,		)
             parser.add_argument(
                 """--dropout"""    ,		type=UpperCAmelCase_    ,		help="""Dropout probability (Optional). Goes into model.config"""    ,		)
             parser.add_argument(
                 """--attention_dropout"""    ,		type=UpperCAmelCase_    ,		help="""Attention dropout probability (Optional). Goes into model.config"""    ,		)
             parser.add_argument("""--learning_rate"""    ,		default=5E-5    ,		type=UpperCAmelCase_    ,		help="""The initial learning rate for Adam."""							)
             parser.add_argument(
                 """--lr_scheduler"""    ,		default="""linear"""    ,		choices=UpperCAmelCase_    ,		metavar=UpperCAmelCase_    ,		type=UpperCAmelCase_    ,		help="""Learning rate scheduler"""    ,		)
             parser.add_argument("""--weight_decay"""    ,		default=0.0    ,		type=UpperCAmelCase_    ,		help="""Weight decay if we apply some."""							)
             parser.add_argument("""--adam_epsilon"""    ,		default=1E-8    ,		type=UpperCAmelCase_    ,		help="""Epsilon for Adam optimizer."""							)
             parser.add_argument("""--warmup_steps"""    ,		default=0    ,		type=UpperCAmelCase_    ,		help="""Linear warmup over warmup_steps."""							)
             parser.add_argument("""--num_workers"""    ,		default=4    ,		type=UpperCAmelCase_    ,		help="""kwarg passed to DataLoader"""							)
             parser.add_argument("""--num_train_epochs"""    ,		dest="""max_epochs"""    ,		default=3    ,		type=UpperCAmelCase_							)
             parser.add_argument("""--train_batch_size"""    ,		default=32    ,		type=UpperCAmelCase_							)
             parser.add_argument("""--eval_batch_size"""    ,		default=32    ,		type=UpperCAmelCase_							)
             parser.add_argument("""--adafactor"""    ,		action="""store_true"""							)



class     SCREAMING_SNAKE_CASE__     (						pl.Callback		):





       '''simple docstring'''




       def UpperCAmelCase_						(						self       :						List[Any]    ,		lowerCAmelCase__       :						Optional[Any]    ,		lowerCAmelCase__       :						Optional[Any]							)			->    Optional[Any]:
             snake_case__					=   trainer.lr_schedulers[0]['scheduler']
             snake_case__					=   {f'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()							)}
             pl_module.logger.log_metrics(UpperCAmelCase_							)

       def UpperCAmelCase_						(						self       :						Dict    ,		lowerCAmelCase__       :						pl.Trainer    ,		lowerCAmelCase__       :						pl.LightningModule							)			->    Dict:
             rank_zero_info("""***** Validation results *****"""							)
             snake_case__					=   trainer.callback_metrics
             # Log results
             for key in sorted(UpperCAmelCase_							):
                   if key not in ["log", "progress_bar"]:
                         rank_zero_info("""{} = {}\n""".format(UpperCAmelCase_    ,		str(metrics[key]							)							)							)

       def UpperCAmelCase_						(						self       :						Dict    ,		lowerCAmelCase__       :						pl.Trainer    ,		lowerCAmelCase__       :						pl.LightningModule							)			->    Tuple:
             rank_zero_info("""***** Test results *****"""							)
             snake_case__					=   trainer.callback_metrics
             # Log and save results to file
             snake_case__					=   os.path.join(pl_module.hparams.output_dir    ,		"""test_results.txt"""							)
             with open(UpperCAmelCase_    ,		"""w"""							) as writer:
                   for key in sorted(UpperCAmelCase_							):
                         if key not in ["log", "progress_bar"]:
                               rank_zero_info("""{} = {}\n""".format(UpperCAmelCase_    ,		str(metrics[key]							)							)							)
                               writer.write("""{} = {}\n""".format(UpperCAmelCase_    ,		str(metrics[key]							)							)							)





def 			_lowercase    (      __UpperCamelCase     :		Any   ,       __UpperCamelCase     :		List[Any] ):
      parser.add_argument(
          """--output_dir"""   ,       default=_UpperCAmelCase   ,       type=_UpperCAmelCase   ,       required=_UpperCAmelCase   ,       help="""The output directory where the model predictions and checkpoints will be written."""   ,       )
      parser.add_argument(
          """--fp16"""   ,       action="""store_true"""   ,       help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""   ,       )

      parser.add_argument(
          """--fp16_opt_level"""   ,       type=_UpperCAmelCase   ,       default="""O2"""   ,       help=(
              """For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\']."""
              """See details at https://nvidia.github.io/apex/amp.html"""
          )   ,       )
      parser.add_argument("""--n_tpu_cores"""   ,       dest="""tpu_cores"""   ,       type=_UpperCAmelCase )
      parser.add_argument("""--max_grad_norm"""   ,       dest="""gradient_clip_val"""   ,       default=1.0   ,       type=_UpperCAmelCase   ,       help="""Max gradient norm""" )
      parser.add_argument("""--do_train"""   ,       action="""store_true"""   ,       help="""Whether to run training.""" )
      parser.add_argument("""--do_predict"""   ,       action="""store_true"""   ,       help="""Whether to run predictions on the test set.""" )
      parser.add_argument(
          """--gradient_accumulation_steps"""   ,       dest="""accumulate_grad_batches"""   ,       type=_UpperCAmelCase   ,       default=1   ,       help="""Number of updates steps to accumulate before performing a backward/update pass."""   ,       )
      parser.add_argument("""--seed"""   ,       type=_UpperCAmelCase   ,       default=42   ,       help="""random seed for initialization""" )
      parser.add_argument(
          """--data_dir"""   ,       default=_UpperCAmelCase   ,       type=_UpperCAmelCase   ,       required=_UpperCAmelCase   ,       help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""   ,       )





def 			_lowercase    (      __UpperCamelCase     :		Optional[int]   ,       __UpperCamelCase     :		List[str]   ,       __UpperCamelCase     :		List[str]=None   ,       __UpperCamelCase     :		Dict=True   ,       __UpperCamelCase     :		Optional[Any]=[]   ,       __UpperCamelCase     :		Tuple=None   ,       __UpperCamelCase     :		str=None   ,       **__UpperCamelCase     :		List[str]   ,       ):
      pl.seed_everything(args.seed )

      # init model
      snake_case__					=   Path(model.hparams.output_dir )
      odir.mkdir(exist_ok=_UpperCAmelCase )

      # add custom checkpoints
      if checkpoint_callback is None:
            snake_case__					=   pl.callbacks.ModelCheckpoint(
                filepath=args.output_dir   ,       prefix="""checkpoint"""   ,       monitor="""val_loss"""   ,       mode="""min"""   ,       save_top_k=1 )
      if early_stopping_callback:
            extra_callbacks.append(_UpperCAmelCase )
      if logging_callback is None:
            snake_case__					=   LoggingCallback()

      snake_case__					=   {}

      # TODO: remove with PyTorch 1.6 since pl uses native amp
      if args.fpaa:
            snake_case__					=   16
            snake_case__					=   args.fpaa_opt_level

      if args.gpus > 1:
            snake_case__					=   'ddp'

      snake_case__					=   args.accumulate_grad_batches
      snake_case__					=   extra_train_kwargs.get("""accelerator"""   ,       _UpperCAmelCase )
      snake_case__					=   extra_train_kwargs.get("""profiler"""   ,       _UpperCAmelCase )

      snake_case__					=   pl.Trainer.from_argparse_args(
          _UpperCAmelCase   ,       weights_summary=_UpperCAmelCase   ,       callbacks=[logging_callback] + extra_callbacks   ,       logger=_UpperCAmelCase   ,       checkpoint_callback=_UpperCAmelCase   ,       **_UpperCAmelCase   ,       )

      if args.do_train:
            trainer.fit(_UpperCAmelCase )

      return trainer









class     SCREAMING_SNAKE_CASE__     :





       '''simple docstring'''




       def __init__(						self       :						Tuple							)			->    None:
             # numbermap keys are the number and its values are lists of indexes sorted
             # in ascending order
             snake_case__					=   {}
             # indexmap keys are an index and it's values are the number at that index
             snake_case__					=   {}

       def UpperCAmelCase_						(						self       :						Optional[int]    ,		lowerCAmelCase__       :						list | str | range    ,		lowerCAmelCase__       :						int							)			->    list[int]:
             if isinstance(lowerCAmelCase__    ,		(range, str)							):
                   snake_case__					=   list(lowerCAmelCase__							)
             elif not isinstance(lowerCAmelCase__    ,		lowerCAmelCase__							):
                   raise TypeError(
                       """binary_search_delete() only accepts either a list, range or str"""							)

             snake_case__					=   0
             snake_case__					=   len(lowerCAmelCase__							) - 1

             while low <= high:
                   snake_case__					=   (low + high) // 2
                   if array[mid] == item:
                         array.pop(lowerCAmelCase__							)
                         return array
                   elif array[mid] < item:
                         snake_case__					=   mid + 1
                   else:
                         snake_case__					=   mid - 1
             raise ValueError(
                 """Either the item is not in the array or the array was unsorted"""							)

       def UpperCAmelCase_						(						self       :						Optional[Any]    ,		lowerCAmelCase__       :						list | str | range    ,		lowerCAmelCase__       :						int							)			->    list[int]:
             if isinstance(lowerCAmelCase__    ,		(range, str)							):
                   snake_case__					=   list(lowerCAmelCase__							)
             elif not isinstance(lowerCAmelCase__    ,		lowerCAmelCase__							):
                   raise TypeError(
                       """binary_search_insert() only accepts either a list, range or str"""							)

             snake_case__					=   0
             snake_case__					=   len(lowerCAmelCase__							) - 1

             while low <= high:
                   snake_case__					=   (low + high) // 2
                   if array[mid] == index:
                         # If the item already exists in the array,
                         # insert it after the existing item
                         array.insert(mid + 1    ,		lowerCAmelCase__							)
                         return array
                   elif array[mid] < index:
                         snake_case__					=   mid + 1
                   else:
                         snake_case__					=   mid - 1

        # If the item doesn't exist in the array, insert it at the appropriate position
             array.insert(lowerCAmelCase__    ,		lowerCAmelCase__							)
             return array

       def UpperCAmelCase_						(						self       :						str    ,		lowerCAmelCase__       :						int    ,		lowerCAmelCase__       :						int							)			->    None:
             # Remove previous index
             if index in self.indexmap:
                   snake_case__					=   self.indexmap[index]
                   if len(self.numbermap[n]							) == 1:
                         del self.numbermap[n]
                   else:
                         snake_case__					=   self.binary_search_delete(self.numbermap[n]    ,		lowerCAmelCase__							)

        # Set new index
             snake_case__					=   number

             # Number not seen before or empty so insert number value
             if number not in self.numbermap:
                   snake_case__					=   [index]

             # Here we need to perform a binary search insertion in order to insert
             # The item in the correct place
             else:
                   snake_case__					=   self.binary_search_insert(
                       self.numbermap[number]    ,		lowerCAmelCase__							)

       def UpperCAmelCase_						(						self       :						Any    ,		lowerCAmelCase__       :						int							)			->    int:
             return self.numbermap.get(lowerCAmelCase__    ,		[-1]							)[0]


if __name__ == "__main__":
       import doctest

       doctest.testmod()






