


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

import torch

from ...utils import is_npu_available, is_xpu_available
from .config_args import ClusterConfig, default_json_config_file
from .config_utils import SubcommandHelpFormatter


_SCREAMING_SNAKE_CASE			: Optional[int]			      =    "Create a default config file for Accelerate with only a few flags set."
def 			UpperCAmelCase__						(UpperCamelCase_="no"							,UpperCamelCase_ = default_json_config_file							,UpperCamelCase_ = False				):



     """simple docstring"""






     snake_case		 =				Path(A__				)
     path.parent.mkdir(parents=A__							,exist_ok=A__				)
     if path.exists():
          print(
              F'''Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`.'''				)
          return False
     snake_case		 =				mixed_precision.lower()
     if mixed_precision not in ["no", "fp16", "bf16", "fp8"]:
          raise ValueError(
              F'''`mixed_precision` should be one of \'no\', \'fp16\', \'bf16\', or \'fp8\'. Received {mixed_precision}'''				)
     snake_case		 =				{
         '''compute_environment''': '''LOCAL_MACHINE''',
         '''mixed_precision''': mixed_precision,
     }
     if torch.cuda.is_available():
          snake_case		 =				torch.cuda.device_count()
          snake_case		 =				num_gpus
          snake_case		 =				False
          if num_gpus > 1:
               snake_case		 =				'''MULTI_GPU'''
          else:
               snake_case		 =				'''NO'''
     elif is_xpu_available() and use_xpu:
          snake_case		 =				torch.xpu.device_count()
          snake_case		 =				num_xpus
          snake_case		 =				False
          if num_xpus > 1:
               snake_case		 =				'''MULTI_XPU'''
          else:
               snake_case		 =				'''NO'''
     elif is_npu_available():
          snake_case		 =				torch.npu.device_count()
          snake_case		 =				num_npus
          snake_case		 =				False
          if num_npus > 1:
               snake_case		 =				'''MULTI_NPU'''
          else:
               snake_case		 =				'''NO'''
     else:
          snake_case		 =				0
          snake_case		 =				True
          snake_case		 =				1
          snake_case		 =				'''NO'''
     snake_case		 =				ClusterConfig(**A__				)
     config.to_json_file(A__				)
     return path
def 			UpperCAmelCase__						(UpperCamelCase_							,UpperCamelCase_				):



     """simple docstring"""






     snake_case		 =				parser.add_parser('''default'''							,parents=A__							,help=A__							,formatter_class=A__				)
     parser.add_argument(
         '''--config_file'''							,default=A__							,help=(
             '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
             '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
             '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
             '''with \'huggingface\'.'''
         )							,dest='''save_location'''							,)

     parser.add_argument(
         '''--mixed_precision'''							,choices=['''no''', '''fp16''', '''bf16''']							,type=A__							,help='''Whether or not to use mixed precision training. '''
         '''Choose between FP16 and BF16 (bfloat16) training. '''
         '''BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.'''							,default='''no'''							,)
     parser.set_defaults(func=A__				)
     return parser

def 			UpperCAmelCase__						(UpperCamelCase_				):



     """simple docstring"""






     snake_case		 =				write_basic_config(args.mixed_precision							,args.save_location				)
     if config_file:
          print(F'''accelerate configuration saved at {config_file}'''				)







import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowercase_									=   1_6
lowercase_									=   3_2
def      a      (			A__	:   Accelerator      ,   A__	:   int = 16      ,   A__	:   str = "bert-base-cased"     )    ->      Optional[int]:





      """simple docstring"""



      _lowercase	       =AutoTokenizer.from_pretrained(A__     )
      _lowercase	       =load_dataset('glue'      ,   'mrpc'     )

      def tokenize_function(A__	:   Optional[int]     ):
            # max_length=None => use the model max length (it's actually the default)
            _lowercase	       =tokenizer(examples['sentence1']      ,   examples['sentence2']      ,   truncation=A__      ,   max_length=A__     )
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      _lowercase	       =datasets.map(
          A__      ,   batched=A__      ,   remove_columns=['idx', 'sentence1', 'sentence2']      ,   load_from_cache_file=A__     )

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      _lowercase	       =tokenized_datasets.rename_column('label'      ,   'labels'     )

      def collate_fn(A__	:   List[str]     ):
            # On TPU it's best to pad everything to the same length or training will be very slow.
            if accelerator.distributed_type == DistributedType.TPU:
                  return tokenizer.pad(A__      ,   padding='max_length'      ,   max_length=128      ,   return_tensors='pt'     )
            return tokenizer.pad(A__      ,   padding='longest'      ,   return_tensors='pt'     )

      # Instantiate dataloaders.
      _lowercase	       =DataLoader(
          tokenized_datasets['train']      ,   shuffle=A__      ,   collate_fn=A__      ,   batch_size=A__     )
      _lowercase	       =DataLoader(
          tokenized_datasets['validation']      ,   shuffle=A__      ,   collate_fn=A__      ,   batch_size=A__     )

      return train_dataloader, eval_dataloader
def      a      (			A__	:   Optional[Any]      ,   A__	:   Optional[int]      ,   A__	:   List[str]      ,   A__	:   Dict     )    ->      Dict:





      """simple docstring"""



      model.eval()
      _lowercase	       =0
      for step, batch in enumerate(A__     ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device     )
            with torch.no_grad():
                  _lowercase	       =model(**A__     )
            _lowercase	       =outputs.logits.argmax(dim=-1     )
            # It is slightly faster to call this once, than multiple times
            _lowercase       ,       _lowercase	       =accelerator.gather(
                (predictions, batch['labels'])     )  # If we are in a multiprocess environment, the last batch has duplicates
            if accelerator.use_distributed:
                  if step == len(A__     ) - 1:
                        _lowercase	       =predictions[: len(eval_dataloader.dataset     ) - samples_seen]
                        _lowercase	       =references[: len(eval_dataloader.dataset     ) - samples_seen]
                  else:
                        samples_seen += references.shape[0]
            metric.add_batch(
                predictions=A__      ,   references=A__      ,   )

      _lowercase	       =metric.compute()
      return eval_metric["accuracy"]
def      a      (			A__	:   str      ,   A__	:   Optional[Any]     )    ->      Optional[Any]:





      """simple docstring"""



      _lowercase	       =Accelerator()

      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      _lowercase	       =config['lr']
      _lowercase	       =int(config['num_epochs']     )
      _lowercase	       =int(config['seed']     )
      _lowercase	       =int(config['batch_size']     )
      _lowercase	       =args.model_name_or_path

      set_seed(A__     )
      _lowercase       ,       _lowercase	       =get_dataloaders(A__      ,   A__      ,   A__     )

      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      _lowercase	       =AutoModelForSequenceClassification.from_pretrained(A__      ,   return_dict=A__     )

      # Instantiate optimizer
      _lowercase	       =(
          AdamW
          if accelerator.state.deepspeed_plugin is None
          or 'optimizer' not in accelerator.state.deepspeed_plugin.deepspeed_config
          else DummyOptim
      )
      _lowercase	       =optimizer_cls(params=model.parameters()      ,   lr=A__     )

      if accelerator.state.deepspeed_plugin is not None:
            _lowercase	       =accelerator.state.deepspeed_plugin.deepspeed_config[
                'gradient_accumulation_steps'
            ]
      else:
            _lowercase	       =1
      _lowercase	       =(len(A__     ) * num_epochs) // gradient_accumulation_steps

      # Instantiate scheduler
      if (
          accelerator.state.deepspeed_plugin is None
          or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
      ):
            _lowercase	       =get_linear_schedule_with_warmup(
                optimizer=A__      ,   num_warmup_steps=0      ,   num_training_steps=A__      ,   )
      else:
            _lowercase	       =DummyScheduler(A__      ,   total_num_steps=A__      ,   warmup_num_steps=0     )

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      _lowercase       ,       _lowercase       ,       _lowercase       ,       _lowercase       ,       _lowercase	       =accelerator.prepare(
          A__      ,   A__      ,   A__      ,   A__      ,   A__     )

      # We need to keep track of how many total steps we have iterated over
      _lowercase	       =0
      # We also need to keep track of the stating epoch so files are named properly
      _lowercase	       =0
      _lowercase	       =evaluate.load('glue'      ,   'mrpc'     )
      _lowercase	       =num_epochs

      if args.partial_train_epoch is not None:
            _lowercase	       =args.partial_train_epoch

      if args.resume_from_checkpoint:
            accelerator.load_state(args.resume_from_checkpoint     )
            _lowercase	       =args.resume_from_checkpoint.split('epoch_'     )[1]
            _lowercase	       =''
            for char in epoch_string:
                  if char.isdigit():
                        state_epoch_num += char
                  else:
                        break
            _lowercase	       =int(A__     ) + 1
            _lowercase	       =evaluation_loop(A__      ,   A__      ,   A__      ,   A__     )
            accelerator.print('resumed checkpoint performance:'      ,   A__     )
            accelerator.print('resumed checkpoint\'s scheduler\'s lr:'      ,   lr_scheduler.get_lr()[0]     )
            accelerator.print('resumed optimizers\'s lr:'      ,   optimizer.param_groups[0]['lr']     )
            with open(os.path.join(args.output_dir      ,   F'''state_{starting_epoch-1}.json'''     )      ,   'r'     ) as f:
                  _lowercase	       =json.load(A__     )
                  assert resumed_state["accuracy"] == accuracy, "Accuracy mismatch, loading from checkpoint failed"
                  assert (
                      resumed_state["lr"] == lr_scheduler.get_lr()[0]
                  ), "Scheduler learning rate mismatch, loading from checkpoint failed"
                  assert (
                      resumed_state["optimizer_lr"] == optimizer.param_groups[0]["lr"]
                  ), "Optimizer learning rate mismatch, loading from checkpoint failed"
                  assert resumed_state["epoch"] == starting_epoch - 1, "Epoch mismatch, loading from checkpoint failed"
                  return

    # Now we train the model
      _lowercase	       ={}
      for epoch in range(A__      ,   A__     ):
            model.train()
            for step, batch in enumerate(A__     ):
                  _lowercase	       =model(**A__     )
                  _lowercase	       =outputs.loss
                  _lowercase	       =loss / gradient_accumulation_steps
                  accelerator.backward(A__     )
                  if step % gradient_accumulation_steps == 0:
                        optimizer.step()
                        lr_scheduler.step()
                        optimizer.zero_grad()

                  overall_step += 1
            _lowercase	       =F'''epoch_{epoch}'''
            _lowercase	       =os.path.join(args.output_dir      ,   A__     )
            accelerator.save_state(A__     )
            _lowercase	       =evaluation_loop(A__      ,   A__      ,   A__      ,   A__     )
            _lowercase	       =accuracy
            _lowercase	       =lr_scheduler.get_lr()[0]
            _lowercase	       =optimizer.param_groups[0]['lr']
            _lowercase	       =epoch
            _lowercase	       =overall_step
            accelerator.print(F'''epoch {epoch}:'''      ,   A__     )

            accelerator.wait_for_everyone()
            if accelerator.is_main_process:
                  with open(os.path.join(args.output_dir      ,   F'''state_{epoch}.json'''     )      ,   'w'     ) as f:
                        json.dump(A__      ,   A__     )
def      a      (			)    ->      Tuple:





      """simple docstring"""



      _lowercase	       =argparse.ArgumentParser(description='Simple example of training script tracking peak GPU memory usage.'     )
      parser.add_argument(
          '--model_name_or_path'      ,   type=A__      ,   default='bert-base-cased'      ,   help='Path to pretrained model or model identifier from huggingface.co/models.'      ,   required=A__      ,   )
      parser.add_argument(
          '--output_dir'      ,   type=A__      ,   default='.'      ,   help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'      ,   )
      parser.add_argument(
          '--resume_from_checkpoint'      ,   type=A__      ,   default=A__      ,   help='If the training should continue from a checkpoint folder.'      ,   )
      parser.add_argument(
          '--partial_train_epoch'      ,   type=A__      ,   default=A__      ,   help='If passed, the training will stop after this number of epochs.'      ,   )
      parser.add_argument(
          '--num_epochs'      ,   type=A__      ,   default=2      ,   help='Number of train epochs.'      ,   )
      _lowercase	       =parser.parse_args()
      _lowercase	       ={'lr': 2e-5, 'num_epochs': args.num_epochs, 'seed': 42, 'batch_size': 16}

      training_function(A__      ,   A__     )


if __name__ == "__main__":
  main()


