




'''simple docstring'''







import inspect
import tempfile
from collections import OrderedDict, UserDict
from collections.abc import MutableMapping
from contextlib import ExitStack, contextmanager
from dataclasses import fields
from enum import Enum
from typing import Any, ContextManager, List, Tuple

import numpy as np

from .import_utils import is_flax_available, is_tf_available, is_torch_available, is_torch_fx_proxy


if is_flax_available():
   import jax.numpy as jnp


class 			_a  (	UpperCamelCase__ ):
       def __get__(	self						:	Optional[int]   ,				lowercase						:	Tuple   ,				lowercase						:	Tuple=None				):



           '''simple docstring'''

           if obj is None:
               return self
           if self.fget is None:
               raise AttributeError('''unreadable attribute'''				)
           UpperCAmelCase			       =		'''__cached_''' + self.fget.__name__
           UpperCAmelCase			       =		getattr(lowercase_   ,				lowercase_   ,				lowercase_				)
           if cached is None:
               UpperCAmelCase			       =		self.fget(lowercase_				)
               setattr(lowercase_   ,				lowercase_   ,				lowercase_				)
           return cached






def 						snake_case_							(_a	:							Dict     ):
    UpperCAmelCase			       =		val.lower()
    if val in {"y", "yes", "t", "true", "on", "1"}:
        return 1
    if val in {"n", "no", "f", "false", "off", "0"}:
        return 0
    raise ValueError(F"invalid truth value {val!r}"     )





def 						snake_case_							(_a	:							Any     ):
    if is_torch_fx_proxy(_lowerCamelCase     ):
        return True
    if is_torch_available():
        import torch

        if isinstance(_lowerCamelCase    ,   torch.Tensor     ):
            return True
    if is_tf_available():
        import tensorflow as tf

        if isinstance(_lowerCamelCase    ,   tf.Tensor     ):
            return True

    if is_flax_available():
        import jax.numpy as jnp
        from jax.core import Tracer

        if isinstance(_lowerCamelCase    ,   (jnp.ndarray, Tracer)     ):
            return True

    return isinstance(_lowerCamelCase    ,   np.ndarray     )





def 						snake_case_							(_a	:							str     ):
    return isinstance(_lowerCamelCase    ,   np.ndarray     )





def 						snake_case_							(_a	:							Union[str, Any]     ):
    return _is_numpy(_lowerCamelCase     )





def 						snake_case_							(_a	:							Dict     ):
    import torch

    return isinstance(_lowerCamelCase    ,   torch.Tensor     )





def 						snake_case_							(_a	:							Union[str, Any]     ):
    return False if not is_torch_available() else _is_torch(_lowerCamelCase     )





def 						snake_case_							(_a	:							Any     ):
    import torch

    return isinstance(_lowerCamelCase    ,   torch.device     )





def 						snake_case_							(_a	:							int     ):
    return False if not is_torch_available() else _is_torch_device(_lowerCamelCase     )





def 						snake_case_							(_a	:							Optional[Any]     ):
    import torch

    if isinstance(_lowerCamelCase    ,   _lowerCamelCase     ):
        if hasattr(_lowerCamelCase    ,   _lowerCamelCase     ):
            UpperCAmelCase			       =		getattr(_lowerCamelCase    ,   _lowerCamelCase     )
        else:
            return False
    return isinstance(_lowerCamelCase    ,   torch.dtype     )





def 						snake_case_							(_a	:							Any     ):
    return False if not is_torch_available() else _is_torch_dtype(_lowerCamelCase     )





def 						snake_case_							(_a	:							List[Any]     ):
    import tensorflow as tf

    return isinstance(_lowerCamelCase    ,   tf.Tensor     )





def 						snake_case_							(_a	:							List[str]     ):
    return False if not is_tf_available() else _is_tensorflow(_lowerCamelCase     )





def 						snake_case_							(_a	:							Union[str, Any]     ):
    import tensorflow as tf

    # the `is_symbolic_tensor` predicate is only available starting with TF 2.14
    if hasattr(_lowerCamelCase    ,   '''is_symbolic_tensor'''     ):
        return tf.is_symbolic_tensor(_lowerCamelCase     )
    return type(_lowerCamelCase     ) == tf.Tensor





def 						snake_case_							(_a	:							str     ):
    return False if not is_tf_available() else _is_tf_symbolic_tensor(_lowerCamelCase     )





def 						snake_case_							(_a	:							str     ):
    import jax.numpy as jnp  # noqa: F811

    return isinstance(_lowerCamelCase    ,   jnp.ndarray     )





def 						snake_case_							(_a	:							Tuple     ):
    return False if not is_flax_available() else _is_jax(_lowerCamelCase     )





def 						snake_case_							(_a	:							Optional[int]     ):
    if isinstance(_lowerCamelCase    ,   (dict, UserDict)     ):
        return {k: to_py_obj(_lowerCamelCase     ) for k, v in obj.items()}
    elif isinstance(_lowerCamelCase    ,   (list, tuple)     ):
        return [to_py_obj(_lowerCamelCase     ) for o in obj]
    elif is_tf_tensor(_lowerCamelCase     ):
        return obj.numpy().tolist()
    elif is_torch_tensor(_lowerCamelCase     ):
        return obj.detach().cpu().tolist()
    elif is_jax_tensor(_lowerCamelCase     ):
        return np.asarray(_lowerCamelCase     ).tolist()
    elif isinstance(_lowerCamelCase    ,   (np.ndarray, np.number)     ):  # tolist also works on 0d np arrays
        return obj.tolist()
    else:
        return obj





def 						snake_case_							(_a	:							int     ):
    if isinstance(_lowerCamelCase    ,   (dict, UserDict)     ):
        return {k: to_numpy(_lowerCamelCase     ) for k, v in obj.items()}
    elif isinstance(_lowerCamelCase    ,   (list, tuple)     ):
        return np.array(_lowerCamelCase     )
    elif is_tf_tensor(_lowerCamelCase     ):
        return obj.numpy()
    elif is_torch_tensor(_lowerCamelCase     ):
        return obj.detach().cpu().numpy()
    elif is_jax_tensor(_lowerCamelCase     ):
        return np.asarray(_lowerCamelCase     )
    else:
        return obj


class 			_a  (	UpperCamelCase__ ):
       def     A      (	self						:	int				):



           '''simple docstring'''

           UpperCAmelCase			       =		fields(self				)

           # Safety and consistency checks
           if not len(lowercase_				):
               raise ValueError(f"{self.__class__.__name__} has no fields."				)
           if not all(field.default is None for field in class_fields[1:]				):
               raise ValueError(f"{self.__class__.__name__} should not have more than one required field."				)

           UpperCAmelCase			       =		getattr(self   ,				class_fields[0].name				)
           UpperCAmelCase			       =		all(getattr(self   ,				field.name				) is None for field in class_fields[1:]				)

           if other_fields_are_none and not is_tensor(lowercase_				):
               if isinstance(lowercase_   ,				lowercase_				):
                   UpperCAmelCase			       =		first_field.items()
                   UpperCAmelCase			       =		True
               else:
                   try:
                       UpperCAmelCase			       =		iter(lowercase_				)
                       UpperCAmelCase			       =		True
                   except TypeError:
                       UpperCAmelCase			       =		False

            # if we provided an iterator as first field and the iterator is a (key, value) iterator
            # set the associated fields
               if first_field_iterator:
                   for idx, element in enumerate(lowercase_				):
                       if (
                           not isinstance(lowercase_   ,				(list, tuple)				)
                           or not len(lowercase_				) == 2
                           or not isinstance(element[0]   ,				lowercase_				)
                       ):
                           if idx == 0:
                               # If we do not have an iterator of key/values, set it as attribute
                               UpperCAmelCase			       =		first_field
                           else:
                               # If we have a mixed iterator, raise an error
                               raise ValueError(
                                   f"Cannot set key/value for {element}. It needs to be a tuple (key, value)."				)
                           break
                       setattr(self   ,				element[0]   ,				element[1]				)
                       if element[1] is not None:
                           UpperCAmelCase			       =		element[1]
               elif first_field is not None:
                   UpperCAmelCase			       =		first_field
           else:
               for field in class_fields:
                   UpperCAmelCase			       =		getattr(self   ,				field.name				)
                   if v is not None:
                       UpperCAmelCase			       =		v
       def __delitem__(	self						:	List[Any]   ,				*lowercase						:	List[Any]   ,				**lowercase						:	Optional[Any]				):



           '''simple docstring'''

           raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance."				)
       def     A      (	self						:	Tuple   ,				*lowercase						:	int   ,				**lowercase						:	int				):



           '''simple docstring'''

           raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance."				)
       def     A      (	self						:	List[Any]   ,				*lowercase						:	Optional[int]   ,				**lowercase						:	Tuple				):



           '''simple docstring'''

           raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance."				)
       def     A      (	self						:	Dict   ,				*lowercase						:	Optional[int]   ,				**lowercase						:	Any				):



           '''simple docstring'''

           raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance."				)
       def __getitem__(	self						:	Optional[Any]   ,				lowercase						:	Optional[Any]				):



           '''simple docstring'''

           if isinstance(lowercase_   ,				lowercase_				):
               UpperCAmelCase			       =		dict(self.items()				)
               return inner_dict[k]
           else:
               return self.to_tuple()[k]
       def __setattr__(	self						:	Optional[Any]   ,				lowercase						:	Union[str, Any]   ,				lowercase						:	Union[str, Any]				):



           '''simple docstring'''

           if name in self.keys() and value is not None:
               # Don't call self.__setitem__ to avoid recursion errors
               super().__setitem__(lowercase_   ,				lowercase_				)
           super().__setattr__(lowercase_   ,				lowercase_				)
       def __setitem__(	self						:	Tuple   ,				lowercase						:	Optional[int]   ,				lowercase						:	Tuple				):



           '''simple docstring'''

           super().__setitem__(lowercase_   ,				lowercase_				)
           # Don't call self.__setattr__ to avoid recursion errors
           super().__setattr__(lowercase_   ,				lowercase_				)







       def     A      (	self						:	List[Any]				):



           '''simple docstring'''

           return tuple(self[k] for k in self.keys()				)






class 			_a  (	UpperCamelCase__							,							UpperCamelCase__ ):
       @classmethod
       def     A      (	cls						:	Any   ,				lowercase						:	int				):



           '''simple docstring'''

           raise ValueError(
               f"{value} is not a valid {cls.__name__}, please select one of {list(cls._valueamember_map_.keys()				)}"				)






class 			_a  (	UpperCamelCase__ ):
       __a				:     Any   				=					"""longest"""
       __a				:     Union[str, Any]   				=					"""max_length"""
       __a				:     Union[str, Any]   				=					"""do_not_pad"""






class 			_a  (	UpperCamelCase__ ):
       __a				:     List[Any]   				=					"""pt"""
       __a				:     int   				=					"""tf"""
       __a				:     int   				=					"""np"""
       __a				:     Any   				=					"""jax"""






class 			_a  :
       def __init__(	self						:	List[str]   ,				lowercase						:	List[ContextManager]				):



           '''simple docstring'''

           UpperCAmelCase			       =		context_managers
           UpperCAmelCase			       =		ExitStack()
       def __enter__(	self						:	Dict				):



           '''simple docstring'''

           for context_manager in self.context_managers:
               self.stack.enter_context(lowercase_				)







       def __exit__(	self						:	List[Any]   ,				*lowercase						:	Optional[Any]   ,				**lowercase						:	str				):



           '''simple docstring'''

           self.stack.__exit__(*lowercase_   ,				**lowercase_				)






def 						snake_case_							(_a	:							Dict     ):
    UpperCAmelCase			       =		infer_framework(_lowerCamelCase     )
    if framework == "tf":
        UpperCAmelCase			       =		inspect.signature(model_class.call     )  # TensorFlow models
    elif framework == "pt":
        UpperCAmelCase			       =		inspect.signature(model_class.forward     )  # PyTorch models
    else:
        UpperCAmelCase			       =		inspect.signature(model_class.__call__     )  # Flax models

    for p in signature.parameters:
        if p == "return_loss" and signature.parameters[p].default is True:
            return True

    return False





def 						snake_case_							(_a	:							List[str]     ):
    UpperCAmelCase			       =		model_class.__name__
    UpperCAmelCase			       =		infer_framework(_lowerCamelCase     )
    if framework == "tf":
        UpperCAmelCase			       =		inspect.signature(model_class.call     )  # TensorFlow models
    elif framework == "pt":
        UpperCAmelCase			       =		inspect.signature(model_class.forward     )  # PyTorch models
    else:
        UpperCAmelCase			       =		inspect.signature(model_class.__call__     )  # Flax models

    if "QuestionAnswering" in model_name:
        return [p for p in signature.parameters if "label" in p or p in ("start_positions", "end_positions")]
    else:
        return [p for p in signature.parameters if "label" in p]





def 						snake_case_							(_a	:							MutableMapping    ,   _a	:							str = ""    ,   _a	:							str = "."     ):

    def _flatten_dict(_a	:							List[Any]    ,   _a	:							int=""    ,   _a	:							Any="."     ):
        for k, v in d.items():
            UpperCAmelCase			       =		str(_lowerCamelCase     ) + delimiter + str(_lowerCamelCase     ) if parent_key else k
            if v and isinstance(_lowerCamelCase    ,   _lowerCamelCase     ):
                yield from flatten_dict(_lowerCamelCase    ,   _lowerCamelCase    ,   delimiter=_lowerCamelCase     ).items()
            else:
                yield key, v

    return dict(_flatten_dict(_lowerCamelCase    ,   _lowerCamelCase    ,   _lowerCamelCase     )     )





@contextmanager
def 						snake_case_							(_a	:							Union[str, Any]    ,   _a	:							bool = False     ):
    if use_temp_dir:
        with tempfile.TemporaryDirectory() as tmp_dir:
            yield tmp_dir
    else:
        yield working_dir





def 						snake_case_							(_a	:							int    ,   _a	:							Optional[Any]=None     ):
    if is_numpy_array(_lowerCamelCase     ):
        return np.transpose(_lowerCamelCase    ,   axes=_lowerCamelCase     )
    elif is_torch_tensor(_lowerCamelCase     ):
        return array.T if axes is None else array.permute(*_lowerCamelCase     )
    elif is_tf_tensor(_lowerCamelCase     ):
        import tensorflow as tf

        return tf.transpose(_lowerCamelCase    ,   perm=_lowerCamelCase     )
    elif is_jax_tensor(_lowerCamelCase     ):
        return jnp.transpose(_lowerCamelCase    ,   axes=_lowerCamelCase     )
    else:
        raise ValueError(F"Type not supported for transpose: {type(_lowerCamelCase     )}."     )





def 						snake_case_							(_a	:							Dict    ,   _a	:							Any     ):
    if is_numpy_array(_lowerCamelCase     ):
        return np.reshape(_lowerCamelCase    ,   _lowerCamelCase     )
    elif is_torch_tensor(_lowerCamelCase     ):
        return array.reshape(*_lowerCamelCase     )
    elif is_tf_tensor(_lowerCamelCase     ):
        import tensorflow as tf

        return tf.reshape(_lowerCamelCase    ,   _lowerCamelCase     )
    elif is_jax_tensor(_lowerCamelCase     ):
        return jnp.reshape(_lowerCamelCase    ,   _lowerCamelCase     )
    else:
        raise ValueError(F"Type not supported for reshape: {type(_lowerCamelCase     )}."     )





def 						snake_case_							(_a	:							List[str]    ,   _a	:							Optional[Any]=None     ):
    if is_numpy_array(_lowerCamelCase     ):
        return np.squeeze(_lowerCamelCase    ,   axis=_lowerCamelCase     )
    elif is_torch_tensor(_lowerCamelCase     ):
        return array.squeeze() if axis is None else array.squeeze(dim=_lowerCamelCase     )
    elif is_tf_tensor(_lowerCamelCase     ):
        import tensorflow as tf

        return tf.squeeze(_lowerCamelCase    ,   axis=_lowerCamelCase     )
    elif is_jax_tensor(_lowerCamelCase     ):
        return jnp.squeeze(_lowerCamelCase    ,   axis=_lowerCamelCase     )
    else:
        raise ValueError(F"Type not supported for squeeze: {type(_lowerCamelCase     )}."     )





def 						snake_case_							(_a	:							List[str]    ,   _a	:							Dict     ):
    if is_numpy_array(_lowerCamelCase     ):
        return np.expand_dims(_lowerCamelCase    ,   _lowerCamelCase     )
    elif is_torch_tensor(_lowerCamelCase     ):
        return array.unsqueeze(dim=_lowerCamelCase     )
    elif is_tf_tensor(_lowerCamelCase     ):
        import tensorflow as tf

        return tf.expand_dims(_lowerCamelCase    ,   axis=_lowerCamelCase     )
    elif is_jax_tensor(_lowerCamelCase     ):
        return jnp.expand_dims(_lowerCamelCase    ,   axis=_lowerCamelCase     )
    else:
        raise ValueError(F"Type not supported for expand_dims: {type(_lowerCamelCase     )}."     )





def 						snake_case_							(_a	:							List[str]     ):
    if is_numpy_array(_lowerCamelCase     ):
        return np.size(_lowerCamelCase     )
    elif is_torch_tensor(_lowerCamelCase     ):
        return array.numel()
    elif is_tf_tensor(_lowerCamelCase     ):
        import tensorflow as tf

        return tf.size(_lowerCamelCase     )
    elif is_jax_tensor(_lowerCamelCase     ):
        return array.size
    else:
        raise ValueError(F"Type not supported for expand_dims: {type(_lowerCamelCase     )}."     )





def 						snake_case_							(_a	:							List[str]    ,   _a	:							Optional[Any]     ):
    for key, value in auto_map.items():
        if isinstance(_lowerCamelCase    ,   (tuple, list)     ):
            UpperCAmelCase			       =		[F"{repo_id}--{v}" if (v is not None and '''--''' not in v) else v for v in value]
        elif value is not None and "--" not in value:
            UpperCAmelCase			       =		F"{repo_id}--{value}"

    return auto_map







def 						snake_case_							(_a	:							Dict     ):
    for base_class in inspect.getmro(_lowerCamelCase     ):
        UpperCAmelCase			       =		base_class.__module__
        UpperCAmelCase			       =		base_class.__name__
        if module.startswith('''tensorflow'''     ) or module.startswith('''keras'''     ) or name == "TFPreTrainedModel":
            return "tf"
        elif module.startswith('''torch'''     ) or name == "PreTrainedModel":
            return "pt"
        elif module.startswith('''flax'''     ) or module.startswith('''jax'''     ) or name == "FlaxPreTrainedModel":
            return "flax"
    else:
        raise TypeError(F"Could not infer framework from class {model_class}."     )











'''simple docstring'''

import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__lowerCAmelCase							:	Optional[int]			=16
__lowerCAmelCase							:	Tuple			=32





def 						UpperCamelCase				(							_lowerCamelCase					:					Accelerator						,       _lowerCamelCase					:					DatasetDict						,       _lowerCamelCase					:					List[int]						,       _lowerCamelCase					:					List[int]						,       _lowerCamelCase					:					int = 16       ):
    A__				=			AutoTokenizer.from_pretrained("bert-base-cased"       )
    A__				=			DatasetDict(
        {
            "train": dataset["train"].select(_lowerCamelCase       ),
            "validation": dataset["train"].select(_lowerCamelCase       ),
            "test": dataset["validation"],
        }       )

    def tokenize_function(_lowerCamelCase					:					Dict       ):
        # max_length=None => use the model max length (it's actually the default)
        A__				=			tokenizer(examples["sentence1"]						,       examples["sentence2"]						,       truncation=_lowerCamelCase						,       max_length=_lowerCamelCase       )
        return outputs

    # Apply the method we just defined to all the examples in all the splits of the dataset
    # starting with the main process first:
    with accelerator.main_process_first():
        A__				=			datasets.map(
            _lowerCamelCase						,       batched=_lowerCamelCase						,       remove_columns=["idx", "sentence1", "sentence2"]						,       )

    # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
    # transformers library
    A__				=			tokenized_datasets.rename_column("label"						,       "labels"       )

    def collate_fn(_lowerCamelCase					:					Any       ):
        # On TPU it's best to pad everything to the same length or training will be very slow.
        A__				=			1_28 if accelerator.distributed_type == DistributedType.TPU else None
        # When using mixed precision we want round multiples of 8/16
        if accelerator.mixed_precision == "fp8":
            A__				=			16
        elif accelerator.mixed_precision != "no":
            A__				=			8
        else:
            A__				=			None

        return tokenizer.pad(
            _lowerCamelCase						,       padding="longest"						,       max_length=_lowerCamelCase						,       pad_to_multiple_of=_lowerCamelCase						,       return_tensors="pt"						,       )

    # Instantiate dataloaders.
    A__				=			DataLoader(
        tokenized_datasets["train"]						,       shuffle=_lowerCamelCase						,       collate_fn=_lowerCamelCase						,       batch_size=_lowerCamelCase       )
    A__				=			DataLoader(
        tokenized_datasets["validation"]						,       shuffle=_lowerCamelCase						,       collate_fn=_lowerCamelCase						,       batch_size=_lowerCamelCase       )

    A__				=			DataLoader(
        tokenized_datasets["test"]						,       shuffle=_lowerCamelCase						,       collate_fn=_lowerCamelCase						,       batch_size=_lowerCamelCase       )

    return train_dataloader, eval_dataloader, test_dataloader





def 						UpperCamelCase				(							_lowerCamelCase					:					List[Any]						,       _lowerCamelCase					:					str       ):
    # New Code #
    A__				=			[]
    # Download the dataset
    A__				=			load_dataset("glue"						,       "mrpc"       )
    # Create our splits
    A__				=			StratifiedKFold(n_splits=int(args.num_folds       )       )
    # Initialize accelerator
    A__				=			Accelerator(cpu=args.cpu						,       mixed_precision=args.mixed_precision       )
    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
    A__				=			config["lr"]
    A__				=			int(config["num_epochs"]       )
    A__				=			int(config["seed"]       )
    A__				=			int(config["batch_size"]       )

    A__				=			evaluate.load("glue"						,       "mrpc"       )

    # If the batch size is too big we use gradient accumulation
    A__				=			1
    if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
        A__				=			batch_size // MAX_GPU_BATCH_SIZE
        A__				=			MAX_GPU_BATCH_SIZE

    set_seed(_lowerCamelCase       )

    # New Code #
    # Create our folds:
    A__				=			kfold.split(np.zeros(datasets["train"].num_rows       )						,       datasets["train"]["label"]       )
    A__				=			[]
    # Iterate over them
    for i, (train_idxs, valid_idxs) in enumerate(_lowerCamelCase       ):
        A__,	A__,	A__				=			get_fold_dataloaders(
            _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       )
        # Instantiate the model (we build the model here so that the seed also control new weights initialization)
        A__				=			AutoModelForSequenceClassification.from_pretrained("bert-base-cased"						,       return_dict=_lowerCamelCase       )

        # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
        # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
        # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
        A__				=			model.to(accelerator.device       )

        # Instantiate optimizer
        A__				=			AdamW(params=model.parameters()						,       lr=_lowerCamelCase       )

        # Instantiate scheduler
        A__				=			get_linear_schedule_with_warmup(
            optimizer=_lowerCamelCase						,       num_warmup_steps=1_00						,       num_training_steps=(len(_lowerCamelCase       ) * num_epochs) // gradient_accumulation_steps						,       )

        # Prepare everything
        # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
        # prepare method.
        A__,	A__,	A__,	A__,	A__				=			accelerator.prepare(
            _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase       )

        # Now we train the model
        for epoch in range(_lowerCamelCase       ):
            model.train()
            for step, batch in enumerate(_lowerCamelCase       ):
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                batch.to(accelerator.device       )
                A__				=			model(**_lowerCamelCase       )
                A__				=			outputs.loss
                A__				=			loss / gradient_accumulation_steps
                accelerator.backward(_lowerCamelCase       )
                if step % gradient_accumulation_steps == 0:
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()

            model.eval()
            for step, batch in enumerate(_lowerCamelCase       ):
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                batch.to(accelerator.device       )
                with torch.no_grad():
                    A__				=			model(**_lowerCamelCase       )
                A__				=			outputs.logits.argmax(dim=-1       )
                A__,	A__				=			accelerator.gather_for_metrics((predictions, batch["labels"])       )
                metric.add_batch(
                    predictions=_lowerCamelCase						,       references=_lowerCamelCase						,       )

            A__				=			metric.compute()
            # Use accelerator.print to print only on the main process.
            accelerator.print(F"epoch {epoch}:"						,       _lowerCamelCase       )

        # New Code #
        # We also run predictions on the test set at the very end
        A__				=			[]
        for step, batch in enumerate(_lowerCamelCase       ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device       )
            with torch.no_grad():
                A__				=			model(**_lowerCamelCase       )
            A__				=			outputs.logits
            A__,	A__				=			accelerator.gather_for_metrics((predictions, batch["labels"])       )
            fold_predictions.append(predictions.cpu()       )
            if i == 0:
                # We need all of the test predictions
                test_references.append(references.cpu()       )
        # Use accelerator.print to print only on the main process.
        test_predictions.append(torch.cat(_lowerCamelCase						,       dim=0       )       )
        # We now need to release all our memory and get rid of the current model, optimizer, etc
        accelerator.free_memory()
    # New Code #
    # Finally we check the accuracy of our folded results:
    A__				=			torch.cat(_lowerCamelCase						,       dim=0       )
    A__				=			torch.stack(_lowerCamelCase						,       dim=0       ).sum(dim=0       ).div(int(args.num_folds       )       ).argmax(dim=-1       )
    A__				=			metric.compute(predictions=_lowerCamelCase						,       references=_lowerCamelCase       )
    accelerator.print("Average test metrics from all folds:"						,       _lowerCamelCase       )





def 						UpperCamelCase				(							):
    A__				=			argparse.ArgumentParser(description="Simple example of training script."       )
    parser.add_argument(
        "--mixed_precision"						,       type=_lowerCamelCase						,       default=_lowerCamelCase						,       choices=["no", "fp16", "bf16", "fp8"]						,       help="Whether to use mixed precision. Choose"
        "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
        "and an Nvidia Ampere GPU."						,       )
    parser.add_argument("--cpu"						,       action="store_true"						,       help="If passed, will train on the CPU."       )
    # New Code #
    parser.add_argument("--num_folds"						,       type=_lowerCamelCase						,       default=3						,       help="The number of splits to perform across the dataset"       )
    A__				=			parser.parse_args()
    A__				=			{"lr": 2e-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
    training_function(_lowerCamelCase						,       _lowerCamelCase       )


if __name__ == "__main__":
  main()



