


import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCamelCase		=	16
lowerCamelCase		=	32






def   lowerCamelCase_					( _a ):




		"""simple docstring"""






		return int(x / 2**20 )





class       _a							:




	def __enter__(      self		:	List[str]	)-> Optional[int]:
			gc.collect()
			torch.cuda.empty_cache()
			torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
			lowerCAmelCase__     :    str					  =      torch.cuda.memory_allocated()
			lowerCAmelCase__     :    int					  =      psutil.Process()

			lowerCAmelCase__     :    List[Any]					  =      self.cpu_mem_used()
			lowerCAmelCase__     :    Dict					  =      True
			lowerCAmelCase__     :    List[str]					  =      threading.Thread(target=self.peak_monitor_func	)
			lowerCAmelCase__     :    Optional[Any]					  =      True
			peak_monitor_thread.start()
			return self




	def 							UpperCAmelCase__(      self		:	int	)-> Dict:
			return self.process.memory_info().rss




	def 							UpperCAmelCase__(      self		:	str	)-> Dict:
			lowerCAmelCase__     :    Optional[int]					  =      -1

			while True:
					lowerCAmelCase__     :    Tuple					  =      max(self.cpu_mem_used()	,      self.cpu_peak	)

					# can't sleep or will not catch the peak right (this comment is here on purpose)
					# time.sleep(0.001) # 1msec

					if not self.peak_monitoring:
							break




	def __exit__(      self		:	Tuple	,      *_SCREAMING_SNAKE_CASE		:	Optional[Any]	)-> Any:
			lowerCAmelCase__     :    Union[str, Any]					  =      False

			gc.collect()
			torch.cuda.empty_cache()
			lowerCAmelCase__     :    List[Any]					  =      torch.cuda.memory_allocated()
			lowerCAmelCase__     :    Tuple					  =      torch.cuda.max_memory_allocated()
			lowerCAmelCase__     :    Dict					  =      bamb(self.end - self.begin	)
			lowerCAmelCase__     :    Any					  =      bamb(self.peak - self.begin	)

			lowerCAmelCase__     :    Dict					  =      self.cpu_mem_used()
			lowerCAmelCase__     :    List[Any]					  =      bamb(self.cpu_end - self.cpu_begin	)
			lowerCAmelCase__     :    List[str]					  =      bamb(self.cpu_peak - self.cpu_begin	)
			# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
		from accelerate.test_utils.training import mocked_dataloaders

		lowerCamelCase		=	mocked_dataloaders  # noqa: F811






def   lowerCamelCase_					( _a			,     _a ):




		"""simple docstring"""






		if os.environ.get('''TESTING_MOCKED_DATALOADERS'''			,     _a ) == "1":
				lowerCAmelCase__     :    Optional[Any]					  =      2

		# New Code #
		# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
		lowerCAmelCase__     :    List[str]					  =      FullyShardedDataParallelPlugin(
		    state_dict_config=FullStateDictConfig(offload_to_cpu=_a			,     ranka_only=_a )			,     optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_a			,     ranka_only=_a )			,     )

		# Initialize accelerator
		if args.with_tracking:
				lowerCAmelCase__     :    str					  =      Accelerator(
				    cpu=args.cpu			,     mixed_precision=args.mixed_precision			,     log_with='''wandb'''			,     project_dir=args.logging_dir			,     fsdp_plugin=_a			,     )
		else:
				lowerCAmelCase__     :    Dict					  =      Accelerator(fsdp_plugin=_a )
		accelerator.print(accelerator.distributed_type )

		if hasattr(args.checkpointing_steps			,     '''isdigit''' ):
				if args.checkpointing_steps == "epoch":
						lowerCAmelCase__     :    List[str]					  =      args.checkpointing_steps
				elif args.checkpointing_steps.isdigit():
						lowerCAmelCase__     :    Any					  =      int(args.checkpointing_steps )
				else:
						raise ValueError(
						    f'Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.' )
		else:
				lowerCAmelCase__     :    int					  =      None
		# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
		lowerCAmelCase__     :    List[Any]					  =      config['''lr''']
		lowerCAmelCase__     :    Dict					  =      int(config['''num_epochs'''] )
		lowerCAmelCase__     :    Dict					  =      int(config['''seed'''] )
		lowerCAmelCase__     :    int					  =      int(config['''batch_size'''] )

		# We need to initialize the trackers we use, and also store our configuration
		if args.with_tracking:
				lowerCAmelCase__     :    Optional[Any]					  =      vars(_a )
				accelerator.init_trackers('''fsdp_glue_no_trainer'''			,     _a )

		lowerCAmelCase__     :    Dict					  =      AutoTokenizer.from_pretrained(args.model_name_or_path )
		lowerCAmelCase__     :    List[Any]					  =      load_dataset('''glue'''			,     '''mrpc''' )
		lowerCAmelCase__     :    List[Any]					  =      evaluate.load('''glue'''			,     '''mrpc''' )

		def tokenize_function(_a ):
				# max_length=None => use the model max length (it's actually the default)
				lowerCAmelCase__     :    List[Any]					  =      tokenizer(examples['''sentence1''']			,     examples['''sentence2''']			,     truncation=_a			,     max_length=_a )
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		# starting with the main process first:
		with accelerator.main_process_first():
				lowerCAmelCase__     :    Any					  =      datasets.map(
				    _a			,     batched=_a			,     remove_columns=['''idx''', '''sentence1''', '''sentence2''']			,     )

		# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
		# transformers library
		lowerCAmelCase__     :    int					  =      tokenized_datasets.rename_column('''label'''			,     '''labels''' )

		# If the batch size is too big we use gradient accumulation
		lowerCAmelCase__     :    str					  =      1
		if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
				lowerCAmelCase__     :    Any					  =      batch_size // MAX_GPU_BATCH_SIZE
				lowerCAmelCase__     :    Union[str, Any]					  =      MAX_GPU_BATCH_SIZE

		def collate_fn(_a ):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				lowerCAmelCase__     :    Union[str, Any]					  =      128 if accelerator.distributed_type == DistributedType.TPU else None
				# When using mixed precision we want round multiples of 8/16
				if accelerator.mixed_precision == "fp8":
						lowerCAmelCase__     :    Optional[int]					  =      16
				elif accelerator.mixed_precision != "no":
						lowerCAmelCase__     :    Dict					  =      8
				else:
						lowerCAmelCase__     :    List[str]					  =      None

				return tokenizer.pad(
				    _a			,     padding='''longest'''			,     max_length=_a			,     pad_to_multiple_of=_a			,     return_tensors='''pt'''			,     )

		# Instantiate dataloaders.
		lowerCAmelCase__     :    str					  =      DataLoader(
		    tokenized_datasets['''train''']			,     shuffle=_a			,     collate_fn=_a			,     batch_size=_a )
		lowerCAmelCase__     :    Optional[int]					  =      DataLoader(
		    tokenized_datasets['''validation''']			,     shuffle=_a			,     collate_fn=_a			,     batch_size=_a )

		set_seed(_a )

		# Instantiate the model (we build the model here so that the seed also control new weights initialization)
		lowerCAmelCase__     :    Union[str, Any]					  =      AutoModelForSequenceClassification.from_pretrained(
		    args.model_name_or_path			,     return_dict=_a			,     low_cpu_mem_usage=_a )

		# New Code #
		# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
		lowerCAmelCase__     :    List[str]					  =      accelerator.prepare(_a )
		accelerator.print(_a )

		# Instantiate optimizer
		# New Code #
		# For FSDP feature, at present it doesn't support multiple parameter groups,
		# so we need to create a single parameter group for the whole model
		lowerCAmelCase__     :    int					  =      torch.optim.AdamW(params=model.parameters()			,     lr=_a			,     weight_decay=2e-4 )

		# Instantiate scheduler
		lowerCAmelCase__     :    Union[str, Any]					  =      get_linear_schedule_with_warmup(
		    optimizer=_a			,     num_warmup_steps=10			,     num_training_steps=(len(_a ) * num_epochs) // gradient_accumulation_steps			,     )

		# New Code #
		# For FSDP feature, prepare everything except the model as we have already prepared the model
		# before creating the optimizer
		# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
		# prepare method.
		lowerCAmelCase__  ,			lowerCAmelCase__  ,			lowerCAmelCase__  ,			lowerCAmelCase__     :    Any					  =      accelerator.prepare(
		    _a			,     _a			,     _a			,     _a )

		lowerCAmelCase__     :    List[str]					  =      0

		# Potentially load in the weights and states from a previous save
		if args.resume_from_checkpoint:
				if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
						accelerator.print(f'Resumed from checkpoint: {args.resume_from_checkpoint}' )
						accelerator.load_state(args.resume_from_checkpoint )
						lowerCAmelCase__     :    int					  =      os.path.basename(args.resume_from_checkpoint )
				else:
						# Get the most recent checkpoint
						lowerCAmelCase__     :    Any					  =      [f.name for f in os.scandir(os.getcwd() ) if f.is_dir()]
						dirs.sort(key=os.path.getctime )
						lowerCAmelCase__     :    List[str]					  =      dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
				# Extract `epoch_{i}` or `step_{i}`
				lowerCAmelCase__     :    Dict					  =      os.path.splitext(_a )[0]

				if "epoch" in training_difference:
						num_epochs -= int(training_difference.replace('''epoch_'''			,     '''''' ) )
						lowerCAmelCase__     :    Union[str, Any]					  =      None
				else:
						lowerCAmelCase__     :    Optional[int]					  =      int(training_difference.replace('''step_'''			,     '''''' ) )
						num_epochs -= resume_step // len(_a )
						# If resuming by step, we also need to know exactly how far into the DataLoader we went
						lowerCAmelCase__     :    str					  =      (num_epochs * len(_a )) - resume_step

    # Now we train the model
		for epoch in range(_a ):
				# New Code #
				# context manager to track the peak memory usage during the training epoch
				with TorchTracemalloc() as tracemalloc:
						model.train()
						if args.with_tracking:
								lowerCAmelCase__     :    Dict					  =      0
						for step, batch in enumerate(_a ):
								# We need to skip steps until we reach the resumed step
								if args.resume_from_checkpoint and epoch == 0:
										if resume_step is not None and step < resume_step:
												pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
								batch.to(accelerator.device )
								lowerCAmelCase__     :    List[str]					  =      model(**_a )
								lowerCAmelCase__     :    Tuple					  =      outputs.loss
								lowerCAmelCase__     :    List[Any]					  =      loss / gradient_accumulation_steps
								# We keep track of the loss at each epoch
								if args.with_tracking:
										total_loss += loss.detach().float()
								accelerator.backward(_a )
								if step % gradient_accumulation_steps == 0:
										optimizer.step()
										lr_scheduler.step()
										optimizer.zero_grad()
										# accelerator.print(lr_scheduler.get_lr())

								overall_step += 1

								if isinstance(_a			,     _a ):
										lowerCAmelCase__     :    List[Any]					  =      f'step_{overall_step}'
										if overall_step % checkpointing_steps == 0:
												if args.output_dir is not None:
														lowerCAmelCase__     :    str					  =      os.path.join(args.output_dir			,     _a )
												accelerator.save_state(_a )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
				accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin ) ) )
				accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used ) )
				accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked ) )
				accelerator.print(
				    '''Total Peak Memory consumed during the train (max): {}'''.format(
				        tracemalloc.peaked + bamb(tracemalloc.begin ) ) )
				# Logging the peak memory usage of the GPU to the tracker
				if args.with_tracking:
						accelerator.log(
						    {
						        '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin ),
						    }			,     step=_a			,     )

				# New Code #
				# context manager to track the peak memory usage during the evaluation
				with TorchTracemalloc() as tracemalloc:
						model.eval()
						for step, batch in enumerate(_a ):
								# We could avoid this line since we set the accelerator with `device_placement=True`.
								batch.to(accelerator.device )
								with torch.no_grad():
										lowerCAmelCase__     :    Any					  =      model(**_a )
								lowerCAmelCase__     :    List[str]					  =      outputs.logits.argmax(dim=-1 )
								lowerCAmelCase__  ,			lowerCAmelCase__     :    List[str]					  =      accelerator.gather_for_metrics((predictions, batch['''labels''']) )
								metric.add_batch(
								    predictions=_a			,     references=_a			,     )

						lowerCAmelCase__     :    Tuple					  =      metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(f'epoch {epoch}:'			,     _a )
						if args.with_tracking:
								accelerator.log(
								    {
								        '''accuracy''': eval_metric['''accuracy'''],
								        '''f1''': eval_metric['''f1'''],
								        '''train_loss''': total_loss.item() / len(_a ),
								    }			,     step=_a			,     )

						if checkpointing_steps == "epoch":
								lowerCAmelCase__     :    int					  =      f'epoch_{epoch}'
								if args.output_dir is not None:
										lowerCAmelCase__     :    int					  =      os.path.join(args.output_dir			,     _a )
								accelerator.save_state(_a )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
				accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin ) ) )
				accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used ) )
				accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked ) )
				accelerator.print(
				    '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin ) ) )
				# Logging the peak memory usage of the GPU to the tracker
				if args.with_tracking:
						accelerator.log(
						    {
						        '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin ),
						    }			,     step=_a			,     )

		if args.with_tracking:
				accelerator.end_training()






def   lowerCamelCase_					( ):




		"""simple docstring"""






		lowerCAmelCase__     :    Tuple					  =      argparse.ArgumentParser(description='''Simple example of training script.''' )
		parser.add_argument(
		    '''--mixed_precision'''			,     type=_a			,     default=_a			,     choices=['''no''', '''fp16''', '''bf16''', '''fp8''']			,     help='''Whether to use mixed precision. Choose'''
		    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
		    '''and an Nvidia Ampere GPU.'''			,     )
		parser.add_argument('''--cpu'''			,     action='''store_true'''			,     help='''If passed, will train on the CPU.''' )
		parser.add_argument(
		    '''--checkpointing_steps'''			,     type=_a			,     default=_a			,     help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''			,     )
		parser.add_argument(
		    '''--resume_from_checkpoint'''			,     type=_a			,     default=_a			,     help='''If the training should continue from a checkpoint folder.'''			,     )
		parser.add_argument(
		    '''--with_tracking'''			,     action='''store_true'''			,     help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''			,     )
		parser.add_argument(
		    '''--output_dir'''			,     type=_a			,     default='''.'''			,     help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''			,     )
		parser.add_argument(
		    '''--logging_dir'''			,     type=_a			,     default='''logs'''			,     help='''Location on where to store experiment tracking logs`'''			,     )
		parser.add_argument(
		    '''--model_name_or_path'''			,     type=_a			,     help='''Path to pretrained model or model identifier from huggingface.co/models.'''			,     required=_a			,     )
		lowerCAmelCase__     :    str					  =      parser.parse_args()
		lowerCAmelCase__     :    str					  =      {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
		training_function(_a			,     _a )


if __name__ == "__main__":
		main()








import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
		import torch

		from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
		import pyanvml.pyanvml as nvml


lowerCamelCase		=	logging.get_logger(__name__)





class       _a							(       _lowercase):
	_a						:						PyTorchBenchmarkArguments
	_a						:						PretrainedConfig
	_a						:						str   					=							"PyTorch"




	@property
	def 							UpperCAmelCase__(      self		:	Dict	)-> Any:
			return torch.__version__




	def 							UpperCAmelCase__(      self		:	List[Any]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> float:
			lowerCAmelCase__     :    Optional[Any]					  =      self._prepare_inference_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_speed(_inference	)




	def 							UpperCAmelCase__(      self		:	List[str]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> [Memory, Optional[MemorySummary]]:
			lowerCAmelCase__     :    List[Any]					  =      self._prepare_inference_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_memory(_inference	)




	def 							UpperCAmelCase__(      self		:	Tuple	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> float:
			lowerCAmelCase__     :    Any					  =      self._prepare_train_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_speed(_train	)




	def 							UpperCAmelCase__(      self		:	Optional[Any]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> [Memory, Optional[MemorySummary]]:
			lowerCAmelCase__     :    Tuple					  =      self._prepare_train_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_memory(_train	)




	def 							UpperCAmelCase__(      self		:	int	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> Callable[[], None]:
			lowerCAmelCase__     :    int					  =      self.config_dict[model_name]

			if self.args.torchscript:
					lowerCAmelCase__     :    List[Any]					  =      True

			lowerCAmelCase__     :    List[Any]					  =      (
			    hasattr(_SCREAMING_SNAKE_CASE	,      '''architectures'''	)
			    and isinstance(config.architectures	,      _SCREAMING_SNAKE_CASE	)
			    and len(config.architectures	) > 0
			)
			if not self.args.only_pretrain_model and has_model_class_in_config:
					try:
							lowerCAmelCase__     :    Dict					  =      config.architectures[0]
							lowerCAmelCase__     :    Optional[Any]					  =      __import__('''transformers'''	,      fromlist=[model_class]	)
							lowerCAmelCase__     :    Any					  =      getattr(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__     :    List[Any]					  =      model_cls(_SCREAMING_SNAKE_CASE	)
					except ImportError:
							raise ImportError(
							    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
							    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''	)
			else:
					lowerCAmelCase__     :    Union[str, Any]					  =      MODEL_MAPPING[config.__class__](_SCREAMING_SNAKE_CASE	)

			model.eval()
			model.to(self.args.device	)

			# encoder-decoder has vocab size saved differently
			lowerCAmelCase__     :    str					  =      config.vocab_size if hasattr(_SCREAMING_SNAKE_CASE	,      '''vocab_size'''	) else config.encoder.vocab_size
			lowerCAmelCase__     :    Dict					  =      torch.randint(_SCREAMING_SNAKE_CASE	,      (batch_size, sequence_length)	,      dtype=torch.long	,      device=self.args.device	)

			if self.args.fpaa:
					logger.info('''Running training in Mixed Precision...'''	)
					if not self.args.is_gpu:
							raise ValueError('''Mixed precision is possible only for GPU.'''	)
					# amp seems to have memory leaks so that memory usage
					# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
					model.half()

			if self.args.torchscript:
					with torch.no_grad():
							lowerCAmelCase__     :    int					  =      torch.jit.trace(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			else:
					lowerCAmelCase__     :    Optional[int]					  =      model

			def encoder_decoder_forward():
					with torch.no_grad():
							lowerCAmelCase__     :    str					  =      inference_model(_SCREAMING_SNAKE_CASE	,      decoder_input_ids=_SCREAMING_SNAKE_CASE	)
					return outputs

			def encoder_forward():
					with torch.no_grad():
							lowerCAmelCase__     :    List[Any]					  =      inference_model(_SCREAMING_SNAKE_CASE	)
					return outputs

			lowerCAmelCase__     :    Any					  =      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
			return _forward




	def 							UpperCAmelCase__(      self		:	str	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> Callable[[], None]:
			lowerCAmelCase__     :    Optional[int]					  =      self.config_dict[model_name]

			lowerCAmelCase__     :    List[str]					  =      (
			    hasattr(_SCREAMING_SNAKE_CASE	,      '''architectures'''	)
			    and isinstance(config.architectures	,      _SCREAMING_SNAKE_CASE	)
			    and len(config.architectures	) > 0
			)
			if not self.args.only_pretrain_model and has_model_class_in_config:
					try:
							lowerCAmelCase__     :    Optional[Any]					  =      config.architectures[0]
							lowerCAmelCase__     :    List[str]					  =      __import__('''transformers'''	,      fromlist=[model_class]	)
							lowerCAmelCase__     :    str					  =      getattr(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__     :    Union[str, Any]					  =      model_cls(_SCREAMING_SNAKE_CASE	)
					except ImportError:
							raise ImportError(
							    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
							    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''	)
			else:
					lowerCAmelCase__     :    Any					  =      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_SCREAMING_SNAKE_CASE	)

			if self.args.torchscript:
					raise NotImplementedError('''Training for torchscript is currently not implemented'''	)
			else:
					lowerCAmelCase__     :    Union[str, Any]					  =      model

			model.train()
			model.to(self.args.device	)

			# encoder-decoder has vocab size saved differently
			lowerCAmelCase__     :    Union[str, Any]					  =      config.vocab_size if hasattr(_SCREAMING_SNAKE_CASE	,      '''vocab_size'''	) else config.encoder.vocab_size
			lowerCAmelCase__     :    List[str]					  =      torch.randint(_SCREAMING_SNAKE_CASE	,      (batch_size, sequence_length)	,      dtype=torch.long	,      device=self.args.device	)

			if self.args.fpaa:
					logger.info('''Running training in Mixed Precision...'''	)
					if not self.args.is_gpu:
							raise ValueError('''Mixed precision is possible only for GPU.'''	)

					# amp seems to have memory leaks so that memory usage
					# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
					model.half()

			def compute_loss_and_backprob_encoder():
					lowerCAmelCase__     :    Optional[Any]					  =      train_model(_SCREAMING_SNAKE_CASE	,      labels=_SCREAMING_SNAKE_CASE	)[0]
					loss.backward()
					return loss

			def compute_loss_and_backprob_encoder_decoder():
					lowerCAmelCase__     :    List[str]					  =      train_model(_SCREAMING_SNAKE_CASE	,      decoder_input_ids=_SCREAMING_SNAKE_CASE	,      labels=_SCREAMING_SNAKE_CASE	)[0]
					loss.backward()
					return loss

			lowerCAmelCase__     :    List[Any]					  =      (
			    compute_loss_and_backprob_encoder_decoder
			    if config.is_encoder_decoder
			    else compute_loss_and_backprob_encoder
			)
			return _train




	def 							UpperCAmelCase__(      self		:	Union[str, Any]	,      _SCREAMING_SNAKE_CASE		:	List[str]	)-> float:
			try:
					if self.args.is_tpu or self.args.torchscript:
							# run additional 10 times to stabilize compilation for tpu and torchscript
							logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''	)
							timeit.repeat(
							    _SCREAMING_SNAKE_CASE	,      repeat=1	,      number=5	,      )

					# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
					lowerCAmelCase__     :    Tuple					  =      timeit.repeat(
					    _SCREAMING_SNAKE_CASE	,      repeat=self.args.repeat	,      number=10	,      )

					if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
							import torch_xla.debug.metrics as met

							self.print_fn(met.metrics_report()	)

					return min(_SCREAMING_SNAKE_CASE	) / 10.0
			except RuntimeError as e:
					self.print_fn(F'Doesn\'t fit on GPU. {e}'	)
					return "N/A"




	def 							UpperCAmelCase__(      self		:	int	,      _SCREAMING_SNAKE_CASE		:	Callable[[], None]	)-> [Memory, MemorySummary]:
			try:
					if self.args.trace_memory_line_by_line:
							lowerCAmelCase__     :    int					  =      start_memory_tracing('''transformers'''	)

					if self.args.is_tpu:
							# tpu
							raise NotImplementedError(
							    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
							    ''' `--no-memory` or `args.memory=False`'''	)
					elif self.args.is_gpu:
							if not is_pyanvml_available():
									logger.warning(
									    '''py3nvml not installed, we won\'t log GPU memory usage. '''
									    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''	)
									lowerCAmelCase__     :    Union[str, Any]					  =      '''N/A'''
							else:
									logger.info(
									    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
									    ''' on the same GPU.'''	)
									# init nvml
									nvml.nvmlInit()
									func()
									lowerCAmelCase__     :    Optional[int]					  =      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx	)
									lowerCAmelCase__     :    Any					  =      nvml.nvmlDeviceGetMemoryInfo(_SCREAMING_SNAKE_CASE	)
									lowerCAmelCase__     :    int					  =      meminfo.used
									lowerCAmelCase__     :    Tuple					  =      Memory(_SCREAMING_SNAKE_CASE	)
									# shutdown nvml
									nvml.nvmlShutdown()
					else:
							# cpu
							lowerCAmelCase__     :    Union[str, Any]					  =      measure_peak_memory_cpu(_SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__     :    Union[str, Any]					  =      Memory(_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	) else memory_bytes

					if self.args.trace_memory_line_by_line:
							lowerCAmelCase__     :    int					  =      stop_memory_tracing(_SCREAMING_SNAKE_CASE	)
					else:
							lowerCAmelCase__     :    str					  =      None

					return memory, summary
			except RuntimeError as e:
					self.print_fn(F'Doesn\'t fit on GPU. {e}'	)
					return "N/A", None




