code: 
  
  
  
  
  import argparse
  import os
  
  import evaluate
  import torch
  from datasets import load_dataset
  from torch.optim import AdamW
  from torch.utils.data import DataLoader
  from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed
  
  from accelerate import Accelerator, DistributedType
  
  
  ########################################################################
  # This is a fully working simple example to use Accelerate,
  # specifically showcasing the checkpointing capability,
  # and builds off the `nlp_example.py` script.
  #
  # This example trains a Bert base model on GLUE MRPC
  # in any of the following settings (with the same script):
  #   - single CPU or single GPU
  #   - multi GPUS (using PyTorch distributed mode)
  #   - (multi) TPUs
  #   - fp16 (mixed-precision) or fp32 (normal precision)
  #
  # To help focus on the differences in the code, building `DataLoaders`
  # was refactored into its own function.
  # New additions from the base script can be found quickly by
  # looking for the # New Code # tags
  #
  # To run it in each of these various modes, follow the instructions
  # in the readme for examples:
  # https://github.com/huggingface/accelerate/tree/main/examples
  #
  ########################################################################
  
  _A							    =	16
  _A							    =	32
  def 						lowerCAmelCase_       (	__a	,	__a = 16 )       ->			List[str]:
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  							SCREAMING_SNAKE_CASE			: List[str]	     =AutoTokenizer.from_pretrained('''bert-base-cased''' )
  							SCREAMING_SNAKE_CASE			: Tuple	     =load_dataset('''glue'''	,	'''mrpc''' )
  
  							def tokenize_function(__a ):
  														# max_length=None => use the model max length (it's actually the default)
  														SCREAMING_SNAKE_CASE			: Tuple	     =tokenizer(examples['''sentence1''']	,	examples['''sentence2''']	,	truncation=__a	,	max_length=__a )
  														return outputs
  
  							# Apply the method we just defined to all the examples in all the splits of the dataset
  							# starting with the main process first:
  							with accelerator.main_process_first():
  														SCREAMING_SNAKE_CASE			: str	     =datasets.map(
  														    __a	,	batched=__a	,	remove_columns=['''idx''', '''sentence1''', '''sentence2''']	,	)
  
  							# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  							# transformers library
  							SCREAMING_SNAKE_CASE			: List[Any]	     =tokenized_datasets.rename_column('''label'''	,	'''labels''' )
  
  							def collate_fn(__a ):
  														# On TPU it's best to pad everything to the same length or training will be very slow.
  														SCREAMING_SNAKE_CASE			: Optional[Any]	     =128 if accelerator.distributed_type == DistributedType.TPU else None
  														# When using mixed precision we want round multiples of 8/16
  														if accelerator.mixed_precision == "fp8":
  																					SCREAMING_SNAKE_CASE			: Optional[int]	     =16
  														elif accelerator.mixed_precision != "no":
  																					SCREAMING_SNAKE_CASE			: Optional[int]	     =8
  														else:
  																					SCREAMING_SNAKE_CASE			: Any	     =None
  
  														return tokenizer.pad(
  														    __a	,	padding='''longest'''	,	max_length=__a	,	pad_to_multiple_of=__a	,	return_tensors='''pt'''	,	)
  
  							# Instantiate dataloaders.
  							SCREAMING_SNAKE_CASE			: Tuple	     =DataLoader(
  							    tokenized_datasets['''train''']	,	shuffle=__a	,	collate_fn=__a	,	batch_size=__a )
  							SCREAMING_SNAKE_CASE			: int	     =DataLoader(
  							    tokenized_datasets['''validation''']	,	shuffle=__a	,	collate_fn=__a	,	batch_size=__a )
  
  							return train_dataloader, eval_dataloader
  
  
  # For testing only
  if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
  							from accelerate.test_utils.training import mocked_dataloaders
  
  							_A							    =	mocked_dataloaders  # noqa: F811
  def 						lowerCAmelCase_       (	__a	,	__a )       ->			List[str]:
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  							if os.environ.get('''TESTING_MOCKED_DATALOADERS'''	,	__a ) == "1":
  														SCREAMING_SNAKE_CASE			: List[Any]	     =2
  							# Initialize accelerator
  							SCREAMING_SNAKE_CASE			: Optional[int]	     =Accelerator(cpu=args.cpu	,	mixed_precision=args.mixed_precision )
  							# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  							SCREAMING_SNAKE_CASE			: Any	     =config['''lr''']
  							SCREAMING_SNAKE_CASE			: int	     =int(config['''num_epochs'''] )
  							SCREAMING_SNAKE_CASE			: int	     =int(config['''seed'''] )
  							SCREAMING_SNAKE_CASE			: List[Any]	     =int(config['''batch_size'''] )
  
  							# New Code #
  							# Parse out whether we are saving every epoch or after a certain number of batches
  							if hasattr(args.checkpointing_steps	,	'''isdigit''' ):
  														if args.checkpointing_steps == "epoch":
  																					SCREAMING_SNAKE_CASE			: str	     =args.checkpointing_steps
  														elif args.checkpointing_steps.isdigit():
  																					SCREAMING_SNAKE_CASE			: Optional[Any]	     =int(args.checkpointing_steps )
  														else:
  																					raise ValueError(
  																					    f'Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.' )
  							else:
  														SCREAMING_SNAKE_CASE			: Union[str, Any]	     =None
  
  							set_seed(__a )
  
  							SCREAMING_SNAKE_CASE     ,							SCREAMING_SNAKE_CASE			: Optional[Any]	     =get_dataloaders(__a	,	__a )
  							SCREAMING_SNAKE_CASE			: Optional[int]	     =evaluate.load('''glue'''	,	'''mrpc''' )
  
  							# If the batch size is too big we use gradient accumulation
  							SCREAMING_SNAKE_CASE			: List[str]	     =1
  							if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
  														SCREAMING_SNAKE_CASE			: Optional[int]	     =batch_size // MAX_GPU_BATCH_SIZE
  														SCREAMING_SNAKE_CASE			: List[str]	     =MAX_GPU_BATCH_SIZE
  
  							# Instantiate the model (we build the model here so that the seed also control new weights initialization)
  							SCREAMING_SNAKE_CASE			: Tuple	     =AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''	,	return_dict=__a )
  
  							# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  							# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  							# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  							SCREAMING_SNAKE_CASE			: Dict	     =model.to(accelerator.device )
  
  							# Instantiate optimizer
  							SCREAMING_SNAKE_CASE			: str	     =AdamW(params=model.parameters()	,	lr=__a )
  
  							# Instantiate scheduler
  							SCREAMING_SNAKE_CASE			: int	     =get_linear_schedule_with_warmup(
  							    optimizer=__a	,	num_warmup_steps=100	,	num_training_steps=(len(__a ) * num_epochs) // gradient_accumulation_steps	,	)
  
  							# Prepare everything
  							# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  							# prepare method.
  							SCREAMING_SNAKE_CASE     ,							SCREAMING_SNAKE_CASE     ,							SCREAMING_SNAKE_CASE     ,							SCREAMING_SNAKE_CASE     ,							SCREAMING_SNAKE_CASE			: Tuple	     =accelerator.prepare(
  							    __a	,	__a	,	__a	,	__a	,	__a )
  
  							# New Code #
  							# We need to keep track of how many total steps we have iterated over
  							SCREAMING_SNAKE_CASE			: int	     =0
  							# We also need to keep track of the stating epoch so files are named properly
  							SCREAMING_SNAKE_CASE			: Optional[int]	     =0
  
  							# We need to load the checkpoint back in before training here with `load_state`
  							# The total number of epochs is adjusted based on where the state is being loaded from,
  							# as we assume continuation of the same training script
  							if args.resume_from_checkpoint:
  														if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
  																					accelerator.print(f'Resumed from checkpoint: {args.resume_from_checkpoint}' )
  																					accelerator.load_state(args.resume_from_checkpoint )
  																					SCREAMING_SNAKE_CASE			: Tuple	     =os.path.basename(args.resume_from_checkpoint )
  														else:
  																					# Get the most recent checkpoint
  																					SCREAMING_SNAKE_CASE			: List[Any]	     =[f.name for f in os.scandir(os.getcwd() ) if f.is_dir()]
  																					dirs.sort(key=os.path.getctime )
  																					SCREAMING_SNAKE_CASE			: Optional[int]	     =dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
  														# Extract `epoch_{i}` or `step_{i}`
  														SCREAMING_SNAKE_CASE			: List[str]	     =os.path.splitext(__a )[0]
  
  														if "epoch" in training_difference:
  																					SCREAMING_SNAKE_CASE			: Any	     =int(training_difference.replace('''epoch_'''	,	'''''' ) ) + 1
  																					SCREAMING_SNAKE_CASE			: List[str]	     =None
  														else:
  																					SCREAMING_SNAKE_CASE			: Optional[Any]	     =int(training_difference.replace('''step_'''	,	'''''' ) )
  																					SCREAMING_SNAKE_CASE			: Tuple	     =resume_step // len(__a )
  																					resume_step -= starting_epoch * len(__a )
  
      # Now we train the model
  							for epoch in range(__a	,	__a ):
  														model.train()
  														# New Code #
  														if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
  																					# We need to skip steps until we reach the resumed step
  																					SCREAMING_SNAKE_CASE			: Optional[Any]	     =accelerator.skip_first_batches(__a	,	__a )
  																					overall_step += resume_step
  														else:
  																					# After the first iteration though, we need to go back to the original dataloader
  																					SCREAMING_SNAKE_CASE			: List[str]	     =train_dataloader
  														for step, batch in enumerate(__a ):
  																					# We could avoid this line since we set the accelerator with `device_placement=True`.
  																					batch.to(accelerator.device )
  																					SCREAMING_SNAKE_CASE			: str	     =model(**__a )
  																					SCREAMING_SNAKE_CASE			: Optional[int]	     =outputs.loss
  																					SCREAMING_SNAKE_CASE			: List[str]	     =loss / gradient_accumulation_steps
  																					accelerator.backward(__a )
  																					if step % gradient_accumulation_steps == 0:
  																												optimizer.step()
  																												lr_scheduler.step()
  																												optimizer.zero_grad()
  																					# New Code #
  																					overall_step += 1
  
  																					# New Code #
  																					# We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
  																					# These are saved to folders named `step_{overall_step}`
  																					# Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
  																					# If mixed precision was used, will also save a "scalar.bin" file
  																					if isinstance(__a	,	__a ):
  																												SCREAMING_SNAKE_CASE			: int	     =f'step_{overall_step}'
  																												if overall_step % checkpointing_steps == 0:
  																																			if args.output_dir is not None:
  																																										SCREAMING_SNAKE_CASE			: Optional[int]	     =os.path.join(args.output_dir	,	__a )
  																																			accelerator.save_state(__a )
  
  														model.eval()
  														for step, batch in enumerate(__a ):
  																					# We could avoid this line since we set the accelerator with `device_placement=True` (the default).
  																					batch.to(accelerator.device )
  																					with torch.no_grad():
  																												SCREAMING_SNAKE_CASE			: Optional[Any]	     =model(**__a )
  																					SCREAMING_SNAKE_CASE			: Any	     =outputs.logits.argmax(dim=-1 )
  																					SCREAMING_SNAKE_CASE     ,							SCREAMING_SNAKE_CASE			: int	     =accelerator.gather_for_metrics((predictions, batch['''labels''']) )
  																					metric.add_batch(
  																					    predictions=__a	,	references=__a	,	)
  
  														SCREAMING_SNAKE_CASE			: Dict	     =metric.compute()
  														# Use accelerator.print to print only on the main process.
  														accelerator.print(f'epoch {epoch}:'	,	__a )
  
  														# New Code #
  														# We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
  														# These are saved to folders named `epoch_{epoch}`
  														# Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
  														# If mixed precision was used, will also save a "scalar.bin" file
  														if checkpointing_steps == "epoch":
  																					SCREAMING_SNAKE_CASE			: Optional[Any]	     =f'epoch_{epoch}'
  																					if args.output_dir is not None:
  																												SCREAMING_SNAKE_CASE			: int	     =os.path.join(args.output_dir	,	__a )
  																					accelerator.save_state(__a )
  def 						lowerCAmelCase_       (	)       ->			List[Any]:
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  							SCREAMING_SNAKE_CASE			: Dict	     =argparse.ArgumentParser(description='''Simple example of training script.''' )
  							parser.add_argument(
  							    '''--mixed_precision'''	,	type=__a	,	default=__a	,	choices=['''no''', '''fp16''', '''bf16''', '''fp8''']	,	help='''Whether to use mixed precision. Choose'''
  							    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
  							    '''and an Nvidia Ampere GPU.'''	,	)
  							parser.add_argument('''--cpu'''	,	action='''store_true'''	,	help='''If passed, will train on the CPU.''' )
  							parser.add_argument(
  							    '''--checkpointing_steps'''	,	type=__a	,	default=__a	,	help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''	,	)
  							parser.add_argument(
  							    '''--output_dir'''	,	type=__a	,	default='''.'''	,	help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''	,	)
  							parser.add_argument(
  							    '''--resume_from_checkpoint'''	,	type=__a	,	default=__a	,	help='''If the training should continue from a checkpoint folder.'''	,	)
  							SCREAMING_SNAKE_CASE			: str	     =parser.parse_args()
  							SCREAMING_SNAKE_CASE			: Any	     ={'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
  							training_function(__a	,	__a )
  
  
  if __name__ == "__main__":
  							main()
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_torch_available,
  )
  
  
  _A							    =	{"""configuration_regnet""": ["""REGNET_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RegNetConfig"""]}
  
  try:
  							if not is_torch_available():
  														raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  							pass
  else:
  							_A							    =	[
  							    """REGNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
  							    """RegNetForImageClassification""",
  							    """RegNetModel""",
  							    """RegNetPreTrainedModel""",
  							]
  
  try:
  							if not is_tf_available():
  														raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  							pass
  else:
  							_A							    =	[
  							    """TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
  							    """TFRegNetForImageClassification""",
  							    """TFRegNetModel""",
  							    """TFRegNetPreTrainedModel""",
  							]
  
  try:
  							if not is_flax_available():
  														raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  							pass
  else:
  							_A							    =	[
  							    """FlaxRegNetForImageClassification""",
  							    """FlaxRegNetModel""",
  							    """FlaxRegNetPreTrainedModel""",
  							]
  
  
  if TYPE_CHECKING:
  							from .configuration_regnet import REGNET_PRETRAINED_CONFIG_ARCHIVE_MAP, RegNetConfig
  
  							try:
  														if not is_torch_available():
  																					raise OptionalDependencyNotAvailable()
  							except OptionalDependencyNotAvailable:
  														pass
  							else:
  														from .modeling_regnet import (
  														    REGNET_PRETRAINED_MODEL_ARCHIVE_LIST,
  														    RegNetForImageClassification,
  														    RegNetModel,
  														    RegNetPreTrainedModel,
  														)
  
  							try:
  														if not is_tf_available():
  																					raise OptionalDependencyNotAvailable()
  							except OptionalDependencyNotAvailable:
  														pass
  							else:
  														from .modeling_tf_regnet import (
  														    TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST,
  														    TFRegNetForImageClassification,
  														    TFRegNetModel,
  														    TFRegNetPreTrainedModel,
  														)
  
  							try:
  														if not is_flax_available():
  																					raise OptionalDependencyNotAvailable()
  							except OptionalDependencyNotAvailable:
  														pass
  							else:
  														from .modeling_flax_regnet import (
  														    FlaxRegNetForImageClassification,
  														    FlaxRegNetModel,
  														    FlaxRegNetPreTrainedModel,
  														)
  
  
  else:
  							import sys
  
  							_A							    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure)
  
  
  
  
  
  
  