code: 
  
  
  
  '''simple docstring'''
  
  
  
  import argparse
  import os
  
  import evaluate
  import torch
  from datasets import load_dataset
  from torch.optim import AdamW
  from torch.utils.data import DataLoader
  from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed
  
  from accelerate import Accelerator, DistributedType
  
  
  ########################################################################
  # This is a fully working simple example to use Accelerate,
  # specifically showcasing the checkpointing capability,
  # and builds off the `nlp_example.py` script.
  #
  # This example trains a Bert base model on GLUE MRPC
  # in any of the following settings (with the same script):
  #   - single CPU or single GPU
  #   - multi GPUS (using PyTorch distributed mode)
  #   - (multi) TPUs
  #   - fp16 (mixed-precision) or fp32 (normal precision)
  #
  # To help focus on the differences in the code, building `DataLoaders`
  # was refactored into its own function.
  # New additions from the base script can be found quickly by
  # looking for the # New Code # tags
  #
  # To run it in each of these various modes, follow the instructions
  # in the readme for examples:
  # https://github.com/huggingface/accelerate/tree/main/examples
  #
  ########################################################################
  
  a__						:			Dict										=			1_6
  a__						:			Optional[int]										=			3_2
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A = 16       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  AutoTokenizer.from_pretrained("""bert-base-cased"""       )
   __UpperCamelCase				      =  load_dataset("""glue"""							,"""mrpc"""       )
  
   def tokenize_function(__A       ):
    # max_length=None => use the model max length (it's actually the default)
    __UpperCamelCase				      =  tokenizer(examples["""sentence1"""]							,examples["""sentence2"""]							,truncation=__A							,max_length=__A       )
    return outputs
  
   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
    __UpperCamelCase				      =  datasets.map(
        __A							,batched=__A							,remove_columns=["""idx""", """sentence1""", """sentence2"""]							,)
  
   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   __UpperCamelCase				      =  tokenized_datasets.rename_column("""label"""							,"""labels"""       )
  
   def collate_fn(__A       ):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __UpperCamelCase				      =  128 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
     __UpperCamelCase				      =  16
    elif accelerator.mixed_precision != "no":
     __UpperCamelCase				      =  8
    else:
     __UpperCamelCase				      =  None
  
    return tokenizer.pad(
        __A							,padding="""longest"""							,max_length=__A							,pad_to_multiple_of=__A							,return_tensors="""pt"""							,)
  
   # Instantiate dataloaders.
   __UpperCamelCase				      =  DataLoader(
       tokenized_datasets["""train"""]							,shuffle=__A							,collate_fn=__A							,batch_size=__A       )
   __UpperCamelCase				      =  DataLoader(
       tokenized_datasets["""validation"""]							,shuffle=__A							,collate_fn=__A							,batch_size=__A       )
  
   return train_dataloader, eval_dataloader
  
  
  # For testing only
  if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
   from accelerate.test_utils.training import mocked_dataloaders
  
   a__						:			Optional[Any]										=			mocked_dataloaders  # noqa: F811
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A       ):
  
  
  
   '''simple docstring'''
  
  
  
   if os.environ.get("""TESTING_MOCKED_DATALOADERS"""							,__A       ) == "1":
    __UpperCamelCase				      =  2
   # Initialize accelerator
   __UpperCamelCase				      =  Accelerator(cpu=args.cpu							,mixed_precision=args.mixed_precision       )
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   __UpperCamelCase				      =  config["""lr"""]
   __UpperCamelCase				      =  int(config["""num_epochs"""]       )
   __UpperCamelCase				      =  int(config["""seed"""]       )
   __UpperCamelCase				      =  int(config["""batch_size"""]       )
  
   # New Code #
   # Parse out whether we are saving every epoch or after a certain number of batches
   if hasattr(args.checkpointing_steps							,"""isdigit"""       ):
    if args.checkpointing_steps == "epoch":
     __UpperCamelCase				      =  args.checkpointing_steps
    elif args.checkpointing_steps.isdigit():
     __UpperCamelCase				      =  int(args.checkpointing_steps       )
    else:
     raise ValueError(
         f"Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."       )
   else:
    __UpperCamelCase				      =  None
  
   set_seed(__A       )
  
   __UpperCamelCase      ,		__UpperCamelCase				      =  get_dataloaders(__A							,__A       )
   __UpperCamelCase				      =  evaluate.load("""glue"""							,"""mrpc"""       )
  
   # If the batch size is too big we use gradient accumulation
   __UpperCamelCase				      =  1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
    __UpperCamelCase				      =  batch_size // MAX_GPU_BATCH_SIZE
    __UpperCamelCase				      =  MAX_GPU_BATCH_SIZE
  
   # Instantiate the model (we build the model here so that the seed also control new weights initialization)
   __UpperCamelCase				      =  AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""							,return_dict=__A       )
  
   # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
   # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
   # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
   __UpperCamelCase				      =  model.to(accelerator.device       )
  
   # Instantiate optimizer
   __UpperCamelCase				      =  AdamW(params=model.parameters()							,lr=__A       )
  
   # Instantiate scheduler
   __UpperCamelCase				      =  get_linear_schedule_with_warmup(
       optimizer=__A							,num_warmup_steps=100							,num_training_steps=(len(__A       ) * num_epochs) // gradient_accumulation_steps							,)
  
   # Prepare everything
   # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
   # prepare method.
   __UpperCamelCase      ,		__UpperCamelCase      ,		__UpperCamelCase      ,		__UpperCamelCase      ,		__UpperCamelCase				      =  accelerator.prepare(
       __A							,__A							,__A							,__A							,__A       )
  
   # New Code #
   # We need to keep track of how many total steps we have iterated over
   __UpperCamelCase				      =  0
   # We also need to keep track of the stating epoch so files are named properly
   __UpperCamelCase				      =  0
  
   # We need to load the checkpoint back in before training here with `load_state`
   # The total number of epochs is adjusted based on where the state is being loaded from,
   # as we assume continuation of the same training script
   if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
     accelerator.print(f"Resumed from checkpoint: {args.resume_from_checkpoint}"       )
     accelerator.load_state(args.resume_from_checkpoint       )
     __UpperCamelCase				      =  os.path.basename(args.resume_from_checkpoint       )
    else:
     # Get the most recent checkpoint
     __UpperCamelCase				      =  [f.name for f in os.scandir(os.getcwd()       ) if f.is_dir()]
     dirs.sort(key=os.path.getctime       )
     __UpperCamelCase				      =  dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract `epoch_{i}` or `step_{i}`
    __UpperCamelCase				      =  os.path.splitext(__A       )[0]
  
    if "epoch" in training_difference:
     __UpperCamelCase				      =  int(training_difference.replace("""epoch_"""							,""""""       )       ) + 1
     __UpperCamelCase				      =  None
    else:
     __UpperCamelCase				      =  int(training_difference.replace("""step_"""							,""""""       )       )
     __UpperCamelCase				      =  resume_step // len(__A       )
     resume_step -= starting_epoch * len(__A       )
  
      # Now we train the model
   for epoch in range(__A							,__A       ):
    model.train()
    # New Code #
    if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
     # We need to skip steps until we reach the resumed step
     __UpperCamelCase				      =  accelerator.skip_first_batches(__A							,__A       )
     overall_step += resume_step
    else:
     # After the first iteration though, we need to go back to the original dataloader
     __UpperCamelCase				      =  train_dataloader
    for step, batch in enumerate(__A       ):
     # We could avoid this line since we set the accelerator with `device_placement=True`.
     batch.to(accelerator.device       )
     __UpperCamelCase				      =  model(**__A       )
     __UpperCamelCase				      =  outputs.loss
     __UpperCamelCase				      =  loss / gradient_accumulation_steps
     accelerator.backward(__A       )
     if step % gradient_accumulation_steps == 0:
      optimizer.step()
      lr_scheduler.step()
      optimizer.zero_grad()
     # New Code #
     overall_step += 1
  
     # New Code #
     # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
     # These are saved to folders named `step_{overall_step}`
     # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
     # If mixed precision was used, will also save a "scalar.bin" file
     if isinstance(__A							,__A       ):
      __UpperCamelCase				      =  f"step_{overall_step}"
      if overall_step % checkpointing_steps == 0:
       if args.output_dir is not None:
        __UpperCamelCase				      =  os.path.join(args.output_dir							,__A       )
       accelerator.save_state(__A       )
  
    model.eval()
    for step, batch in enumerate(__A       ):
     # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
     batch.to(accelerator.device       )
     with torch.no_grad():
      __UpperCamelCase				      =  model(**__A       )
     __UpperCamelCase				      =  outputs.logits.argmax(dim=-1       )
     __UpperCamelCase      ,		__UpperCamelCase				      =  accelerator.gather_for_metrics((predictions, batch["""labels"""])       )
     metric.add_batch(
         predictions=__A							,references=__A							,)
  
    __UpperCamelCase				      =  metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(f"epoch {epoch}:"							,__A       )
  
    # New Code #
    # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
    # These are saved to folders named `epoch_{epoch}`
    # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
    # If mixed precision was used, will also save a "scalar.bin" file
    if checkpointing_steps == "epoch":
     __UpperCamelCase				      =  f"epoch_{epoch}"
     if args.output_dir is not None:
      __UpperCamelCase				      =  os.path.join(args.output_dir							,__A       )
     accelerator.save_state(__A       )
  
  
  
  
  
  
  
  def _lowercase   (  ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  argparse.ArgumentParser(description="""Simple example of training script."""       )
   parser.add_argument(
       """--mixed_precision"""							,type=__A							,default=__A							,choices=["""no""", """fp16""", """bf16""", """fp8"""]							,help="""Whether to use mixed precision. Choose"""
       """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
       """and an Nvidia Ampere GPU."""							,)
   parser.add_argument("""--cpu"""							,action="""store_true"""							,help="""If passed, will train on the CPU."""       )
   parser.add_argument(
       """--checkpointing_steps"""							,type=__A							,default=__A							,help="""Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch."""							,)
   parser.add_argument(
       """--output_dir"""							,type=__A							,default="""."""							,help="""Optional save directory where all checkpoint folders will be stored. Default is the current working directory."""							,)
   parser.add_argument(
       """--resume_from_checkpoint"""							,type=__A							,default=__A							,help="""If the training should continue from a checkpoint folder."""							,)
   __UpperCamelCase				      =  parser.parse_args()
   __UpperCamelCase				      =  {"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
   training_function(__A							,__A       )
  
  
  if __name__ == "__main__":
   main()
  
  
  
  
  
  
style_context: 
  
  
  
  '''simple docstring'''
  
  
  
  import argparse
  
  from transformers import TaConfig, TaForConditionalGeneration, load_tf_weights_in_ta
  from transformers.utils import logging
  
  
  logging.set_verbosity_info()
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A							,__A       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  TaConfig.from_json_file(__A       )
   print(f"Building PyTorch model from configuration: {config}"       )
   __UpperCamelCase				      =  TaForConditionalGeneration(__A       )
  
   # Load weights from tf checkpoint
   load_tf_weights_in_ta(__A							,__A							,__A       )
  
   # Save pytorch-model
   print(f"Save PyTorch model to {pytorch_dump_path}"       )
   model.save_pretrained(__A       )
  
  
  if __name__ == "__main__":
   a__						:			int										=			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
   )
   parser.add_argument(
       '--config_file',
       default=None,
       type=str,
       required=True,
       help=(
           'The config json file corresponding to the pre-trained T5 model. \nThis specifies the model architecture.'
       ),
   )
   parser.add_argument(
       '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
   )
   a__						:			Optional[Any]										=			parser.parse_args()
   convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)
  
  
  
  
  
  