



"""simple docstring"""







from dataclasses import dataclass, field
from typing import Tuple

from ..utils import cached_property, is_torch_available, is_torch_tpu_available, logging, requires_backends
from .benchmark_args_utils import BenchmarkArguments


if is_torch_available():
     import torch

if is_torch_tpu_available(check_device=False):
     import torch_xla.core.xla_model as xm


lowercase__		:				int          =	logging.get_logger(__name__)







@dataclass
class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     [
          """no_inference""",
          """no_cuda""",
          """no_tpu""",
          """no_speed""",
          """no_memory""",
          """no_env_print""",
          """no_multi_process""",
      ]



      def __init__(      self				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      Dict       ):
        for deprecated_arg in self.deprecated_args:
          if deprecated_arg in kwargs:
            lowerCAmelCase_   :				str								= deprecated_arg[3:]
            setattr(self    ,			SCREAMING_SNAKE_CASE_    ,			not kwargs.pop(SCREAMING_SNAKE_CASE_       )       )
            logger.warning(
                F"{deprecated_arg} is depreciated. Please use --no_{positive_arg} or"
                F" {positive_arg}={kwargs[positive_arg]}"       )

        lowerCAmelCase_   :				Optional[int]								= kwargs.pop('torchscript'    ,			self.torchscript       )
        lowerCAmelCase_   :				Any								= kwargs.pop('torch_xla_tpu_print_metrics'    ,			self.torch_xla_tpu_print_metrics       )
        lowerCAmelCase_   :				Union[str, Any]								= kwargs.pop('fp16_opt_level'    ,			self.fpaa_opt_level       )
        super().__init__(**SCREAMING_SNAKE_CASE_       )

      _SCREAMING_SNAKE_CASE  				=     field(default=lowercase_,					metadata={"""help""": """Trace the models using torchscript"""}  )
      _SCREAMING_SNAKE_CASE  				=     field(default=lowercase_,					metadata={"""help""": """Print Xla/PyTorch tpu metrics"""}  )
      _SCREAMING_SNAKE_CASE  				=     field(
          default="""O1""",					metadata={
              """help""": (
                  """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. """
                  """See details at https://nvidia.github.io/apex/amp.html"""
              )
          },					)



      @cached_property
      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        requires_backends(self    ,			['torch']       )
        logger.info('PyTorch: setting up devices'       )
        if not self.cuda:
          lowerCAmelCase_   :				Union[str, Any]								= torch.device('cpu'       )
          lowerCAmelCase_   :				Union[str, Any]								= 0
        elif is_torch_tpu_available():
          lowerCAmelCase_   :				Any								= xm.xla_device()
          lowerCAmelCase_   :				List[Any]								= 0
        else:
          lowerCAmelCase_   :				Optional[int]								= torch.device('cuda' if torch.cuda.is_available() else 'cpu'       )
          lowerCAmelCase_   :				Tuple								= torch.cuda.device_count()
        return device, n_gpu



      @property
      def      SCREAMING_SNAKE_CASE__ (      self				:      str       ):
        return is_torch_tpu_available() and self.tpu



      @property
      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[int]       ):
        requires_backends(self    ,			['torch']       )
        # TODO(PVP): currently only single GPU is supported
        return torch.cuda.current_device()



      @property
      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        requires_backends(self    ,			['torch']       )
        return self._setup_devices[0]



      @property
      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        requires_backends(self    ,			['torch']       )
        return self._setup_devices[1]





      @property
      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        return self.n_gpu > 0










"""simple docstring"""







import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
     import tensorflow as tf
     from tensorflow.python.eager import context
     from tensorflow.python.framework import ops

     from transformers import GradientAccumulator, create_optimizer







@require_tf
class       UpperCamelCase__   (	unittest.TestCase  ):




      """simple docstring"""




      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]    ,			SCREAMING_SNAKE_CASE_				:      str    ,			SCREAMING_SNAKE_CASE_				:      List[Any]    ,			SCREAMING_SNAKE_CASE_				:      List[str]       ):
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			len(SCREAMING_SNAKE_CASE_       )       )
        for a, b in zip(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       ):
          self.assertAlmostEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			delta=SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        lowerCAmelCase_   :				int								= GradientAccumulator()
        accumulator([tf.constant([1.0, 2.0]       )]       )
        accumulator([tf.constant([-2.0, 1.0]       )]       )
        accumulator([tf.constant([-1.0, 2.0]       )]       )
        with self.assertRaises(SCREAMING_SNAKE_CASE_       ):
          accumulator([tf.constant([1.0, 1.0]       ), tf.constant([2.0, 2.0]       )]       )
        self.assertEqual(accumulator.step    ,			3       )
        self.assertEqual(len(accumulator.gradients       )    ,			1       )
        self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()    ,			[-2.0, 5.0]    ,			tol=1E-2       )
        accumulator.reset()
        self.assertEqual(accumulator.step    ,			0       )
        self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()    ,			[0.0, 0.0]    ,			tol=1E-2       )





      def      SCREAMING_SNAKE_CASE__ (      self				:      str       ):
        lowerCAmelCase_   :				Optional[int]								= None
        ops.enable_eager_execution_internal()
        lowerCAmelCase_   :				str								= tf.config.list_physical_devices('CPU'       )
        if len(SCREAMING_SNAKE_CASE_       ) == 1:
          tf.config.set_logical_device_configuration(
              physical_devices[0]    ,			[tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]       )
        lowerCAmelCase_   :				Dict								= tf.config.list_logical_devices(device_type='CPU'       )
        lowerCAmelCase_   :				Optional[int]								= tf.distribute.MirroredStrategy(devices=devices[:2]       )

        with strategy.scope():
          lowerCAmelCase_   :				Union[str, Any]								= GradientAccumulator()
          lowerCAmelCase_   :				int								= tf.Variable([4.0, 3.0]       )
          lowerCAmelCase_						,lowerCAmelCase_   :				Optional[int]								= create_optimizer(5E-5    ,			1_0    ,			5       )
          lowerCAmelCase_   :				Union[str, Any]								= tf.Variable([0.0, 0.0]    ,			trainable=SCREAMING_SNAKE_CASE_       )

        def accumulate_on_replica(SCREAMING_SNAKE_CASE_				:      Optional[int]       ):
          accumulator([gradient]       )

        def apply_on_replica():
          optimizer.apply_gradients(list(zip(accumulator.gradients    ,			[variable]       )       )       )

        @tf.function
        def accumulate(SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			SCREAMING_SNAKE_CASE_				:      Union[str, Any]       ):
          with strategy.scope():
            lowerCAmelCase_   :				Tuple								= strategy.experimental_local_results(SCREAMING_SNAKE_CASE_       )
            local_variables[0].assign(SCREAMING_SNAKE_CASE_       )
            local_variables[1].assign(SCREAMING_SNAKE_CASE_       )
            strategy.run(SCREAMING_SNAKE_CASE_    ,			args=(gradient_placeholder,)       )

        @tf.function
        def apply_grad():
          with strategy.scope():
            strategy.run(SCREAMING_SNAKE_CASE_       )

        def _check_local_values(SCREAMING_SNAKE_CASE_				:      Union[str, Any]    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]       ):
          lowerCAmelCase_   :				List[Any]								= strategy.experimental_local_results(accumulator._gradients[0]       )
          self.assertListAlmostEqual(values[0].value()    ,			SCREAMING_SNAKE_CASE_    ,			tol=1E-2       )
          self.assertListAlmostEqual(values[1].value()    ,			SCREAMING_SNAKE_CASE_    ,			tol=1E-2       )

        accumulate([1.0, 2.0]    ,			[-1.0, 1.0]       )
        accumulate([3.0, -1.0]    ,			[-1.0, -1.0]       )
        accumulate([-2.0, 2.0]    ,			[3.0, -2.0]       )
        self.assertEqual(accumulator.step    ,			3       )
        _check_local_values([2.0, 3.0]    ,			[1.0, -2.0]       )
        apply_grad()
        self.assertListAlmostEqual(variable.value()    ,			[4.0, 3.0]    ,			tol=1E-2       )
        accumulator.reset()
        self.assertEqual(accumulator.step    ,			0       )
        _check_local_values([0.0, 0.0]    ,			[0.0, 0.0]       )





