


'''simple docstring'''
import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(4_2)

UpperCamelCase__    					=      '''bert-base-cased'''
UpperCamelCase__    					=      '''fp16'''
UpperCamelCase__    					=      '''bf16'''
UpperCamelCase__    					=      [FPaa, BFaa]




@require_fsdp
@require_cuda
class 			lowerCamelCase_   (	__a      ):






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          super().setUp()

          UpperCAmelCase__			:		Tuple													=				dict(
              ACCELERATE_USE_FSDP='''true'''						,						MASTER_ADDR='''localhost'''						,						MASTER_PORT='''10999'''						,						RANK='''0'''						,						LOCAL_RANK='''0'''						,						WORLD_SIZE='''1'''						,						)






      def lowercase_							(					self				:    List[Any] ):


          '''simple docstring'''


          from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

          for i, strategy in enumerate(_A ):
              UpperCAmelCase__			:		str													=				self.dist_env.copy()
              UpperCAmelCase__			:		int													=				f"""{i + 1}"""
              UpperCAmelCase__			:		Any													=				strategy
              with mockenv_context(**_A ):
                  UpperCAmelCase__			:		str													=				FullyShardedDataParallelPlugin()
                  self.assertEqual(fsdp_plugin.sharding_strategy						,						ShardingStrategy(i + 1 ) )






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

          for i, prefetch_policy in enumerate(_A ):
              UpperCAmelCase__			:		List[str]													=				self.dist_env.copy()
              UpperCAmelCase__			:		List[str]													=				prefetch_policy
              with mockenv_context(**_A ):
                  UpperCAmelCase__			:		Union[str, Any]													=				FullyShardedDataParallelPlugin()
                  if prefetch_policy == "NO_PREFETCH":
                      self.assertIsNone(fsdp_plugin.backward_prefetch )
                  else:
                      self.assertEqual(fsdp_plugin.backward_prefetch						,						BackwardPrefetch(i + 1 ) )






      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

          for i, state_dict_type in enumerate(_A ):
              UpperCAmelCase__			:		int													=				self.dist_env.copy()
              UpperCAmelCase__			:		int													=				state_dict_type
              with mockenv_context(**_A ):
                  UpperCAmelCase__			:		Dict													=				FullyShardedDataParallelPlugin()
                  self.assertEqual(fsdp_plugin.state_dict_type						,						StateDictType(i + 1 ) )
                  if state_dict_type == "FULL_STATE_DICT":
                      self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu )
                      self.assertTrue(fsdp_plugin.state_dict_config.ranka_only )






      def lowercase_							(					self				:    Optional[Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		int													=				AutoModel.from_pretrained(_A )
          for policy in FSDP_AUTO_WRAP_POLICY:
              UpperCAmelCase__			:		Dict													=				self.dist_env.copy()
              UpperCAmelCase__			:		List[Any]													=				policy
              if policy == "TRANSFORMER_BASED_WRAP":
                  UpperCAmelCase__			:		Dict													=				'''BertLayer'''
              elif policy == "SIZE_BASED_WRAP":
                  UpperCAmelCase__			:		List[Any]													=				'''2000'''
              with mockenv_context(**_A ):
                  UpperCAmelCase__			:		List[Any]													=				FullyShardedDataParallelPlugin()
                  fsdp_plugin.set_auto_wrap_policy(_A )
                  if policy == "NO_WRAP":
                      self.assertIsNone(fsdp_plugin.auto_wrap_policy )
                  else:
                      self.assertIsNotNone(fsdp_plugin.auto_wrap_policy )

          UpperCAmelCase__			:		List[str]													=				self.dist_env.copy()
          UpperCAmelCase__			:		Dict													=				'''TRANSFORMER_BASED_WRAP'''
          UpperCAmelCase__			:		Optional[Any]													=				'''T5Layer'''
          with mockenv_context(**_A ):
              UpperCAmelCase__			:		Optional[int]													=				FullyShardedDataParallelPlugin()
              with self.assertRaises(_A ) as cm:
                  fsdp_plugin.set_auto_wrap_policy(_A )
              self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception ) )

          UpperCAmelCase__			:		Any													=				self.dist_env.copy()
          UpperCAmelCase__			:		Any													=				'''SIZE_BASED_WRAP'''
          UpperCAmelCase__			:		Any													=				'''0'''
          with mockenv_context(**_A ):
              UpperCAmelCase__			:		int													=				FullyShardedDataParallelPlugin()
              fsdp_plugin.set_auto_wrap_policy(_A )
              self.assertIsNone(fsdp_plugin.auto_wrap_policy )






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
          from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

          for mp_dtype in dtypes:
              UpperCAmelCase__			:		List[str]													=				self.dist_env.copy()
              UpperCAmelCase__			:		Dict													=				mp_dtype
              with mockenv_context(**_A ):
                  UpperCAmelCase__			:		Dict													=				Accelerator()
                  if mp_dtype == "fp16":
                      UpperCAmelCase__			:		Any													=				torch.floataa
                  elif mp_dtype == "bf16":
                      UpperCAmelCase__			:		int													=				torch.bfloataa
                  UpperCAmelCase__			:		List[str]													=				MixedPrecision(param_dtype=_A						,						reduce_dtype=_A						,						buffer_dtype=_A )
                  self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy						,						_A )
                  if mp_dtype == FPaa:
                      self.assertTrue(isinstance(accelerator.scaler						,						_A ) )
                  elif mp_dtype == BFaa:
                      self.assertIsNone(accelerator.scaler )
                  AcceleratorState._reset_state(_A )






      def lowercase_							(					self				:    Optional[Any] ):


          '''simple docstring'''


          from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

          for flag in [True, False]:
              UpperCAmelCase__			:		str													=				self.dist_env.copy()
              UpperCAmelCase__			:		Any													=				str(_A ).lower()
              with mockenv_context(**_A ):
                  UpperCAmelCase__			:		Any													=				FullyShardedDataParallelPlugin()
                  self.assertEqual(fsdp_plugin.cpu_offload						,						CPUOffload(offload_params=_A ) )




@require_fsdp
@require_multi_gpu
@slow
class 			lowerCamelCase_   (	__a      ):






      def lowercase_							(					self				:    Optional[Any] ):


          '''simple docstring'''


          super().setUp()
          UpperCAmelCase__			:		Tuple													=				0.8_2
          UpperCAmelCase__			:		int													=				[
              '''fsdp_shard_grad_op_transformer_based_wrap''',
              '''fsdp_full_shard_transformer_based_wrap''',
          ]
          UpperCAmelCase__			:		str													=				{
              '''multi_gpu_fp16''': 3_200,
              '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2_000,
              '''fsdp_full_shard_transformer_based_wrap_fp16''': 1_900,
              # Disabling below test as it overwhelms the RAM memory usage
              # on CI self-hosted runner leading to tests getting killed.
              # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
          }
          UpperCAmelCase__			:		List[str]													=				160
          UpperCAmelCase__			:		Optional[Any]													=				160

          UpperCAmelCase__			:		str													=				inspect.getfile(accelerate.test_utils )
          UpperCAmelCase__			:		Optional[int]													=				os.path.sep.join(mod_file.split(os.path.sep )[:-1] + ['''scripts''', '''external_deps'''] )






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				os.path.join(self.test_scripts_folder						,						'''test_performance.py''' )
          UpperCAmelCase__			:		List[Any]													=				['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
          for config in self.performance_configs:
              UpperCAmelCase__			:		Any													=				cmd.copy()
              for i, strategy in enumerate(_A ):
                  if strategy.lower() in config:
                      cmd_config.append(f"""--fsdp_sharding_strategy={i+1}""" )
                      break

              if "fp32" in config:
                  cmd_config.append('''--mixed_precision=no''' )
              else:
                  cmd_config.append('''--mixed_precision=fp16''' )

              if "cpu_offload" in config:
                  cmd_config.append('''--fsdp_offload_params=True''' )

              for policy in FSDP_AUTO_WRAP_POLICY:
                  if policy.lower() in config:
                      cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}""" )
                      break

              if policy == "TRANSFORMER_BASED_WRAP":
                  cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer''' )
              elif policy == "SIZE_BASED_WRAP":
                  cmd_config.append('''--fsdp_min_num_params=2000''' )

              cmd_config.extend(
                  [
                      self.test_file_path,
                      f"""--output_dir={self.tmpdir}""",
                      f"""--performance_lower_bound={self.performance_lower_bound}""",
                  ] )
              with patch_environment(omp_num_threads=1 ):
                  execute_subprocess_async(_A						,						env=os.environ.copy() )






      def lowercase_							(					self				:    List[Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				os.path.join(self.test_scripts_folder						,						'''test_checkpointing.py''' )
          UpperCAmelCase__			:		Dict													=				[
              '''accelerate''',
              '''launch''',
              '''--num_processes=2''',
              '''--num_machines=1''',
              '''--machine_rank=0''',
              '''--use_fsdp''',
              '''--mixed_precision=fp16''',
              '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
          ]

          for i, strategy in enumerate(_A ):
              UpperCAmelCase__			:		List[str]													=				cmd.copy()
              cmd_config.append(f"""--fsdp_sharding_strategy={i+1}""" )
              if strategy != "FULL_SHARD":
                  continue
              UpperCAmelCase__			:		Tuple													=				len(_A )
              for state_dict_type in FSDP_STATE_DICT_TYPE:
                  UpperCAmelCase__			:		int													=				cmd_config[:state_dict_config_index]
                  cmd_config.append(f"""--fsdp_state_dict_type={state_dict_type}""" )
                  cmd_config.extend(
                      [
                          self.test_file_path,
                          f"""--output_dir={self.tmpdir}""",
                          '''--partial_train_epoch=1''',
                      ] )
                  with patch_environment(omp_num_threads=1 ):
                      execute_subprocess_async(_A						,						env=os.environ.copy() )

                  UpperCAmelCase__			:		List[Any]													=				cmd_config[:-1]
                  UpperCAmelCase__			:		Union[str, Any]													=				os.path.join(self.tmpdir						,						'''epoch_0''' )
                  cmd_config.extend(
                      [
                          f"""--resume_from_checkpoint={resume_from_checkpoint}""",
                      ] )
                  with patch_environment(omp_num_threads=1 ):
                      execute_subprocess_async(_A						,						env=os.environ.copy() )






      def lowercase_							(					self				:    List[str] ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				os.path.join(self.test_scripts_folder						,						'''test_peak_memory_usage.py''' )
          UpperCAmelCase__			:		Dict													=				[
              '''accelerate''',
              '''launch''',
              '''--num_processes=2''',
              '''--num_machines=1''',
              '''--machine_rank=0''',
          ]
          for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
              UpperCAmelCase__			:		Dict													=				cmd.copy()
              if "fp16" in spec:
                  cmd_config.extend(['''--mixed_precision=fp16'''] )
              else:
                  cmd_config.extend(['''--mixed_precision=no'''] )

              if "multi_gpu" in spec:
                  continue
              else:
                  cmd_config.extend(['''--use_fsdp'''] )
                  for i, strategy in enumerate(_A ):
                      if strategy.lower() in spec:
                          cmd_config.append(f"""--fsdp_sharding_strategy={i+1}""" )
                          break

                  if "cpu_offload" in spec:
                      cmd_config.append('''--fsdp_offload_params=True''' )

                  for policy in FSDP_AUTO_WRAP_POLICY:
                      if policy.lower() in spec:
                          cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}""" )
                          break

                  if policy == "TRANSFORMER_BASED_WRAP":
                      cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer''' )
                  elif policy == "SIZE_BASED_WRAP":
                      cmd_config.append('''--fsdp_min_num_params=2000''' )

              cmd_config.extend(
                  [
                      self.test_file_path,
                      f"""--output_dir={self.tmpdir}""",
                      f"""--peak_memory_upper_bound={peak_mem_upper_bound}""",
                      f"""--n_train={self.n_train}""",
                      f"""--n_val={self.n_val}""",
                  ] )
              with patch_environment(omp_num_threads=1 ):
                  execute_subprocess_async(_A						,						env=os.environ.copy() )




'''simple docstring'''
import pprint

import requests

UpperCamelCase__    					=      '''https://zenquotes.io/api'''





def 					a__       (					)    ->	list:
    return requests.get(API_ENDPOINT_URL + '''/today'''						).json()





def 					a__       (					)    ->	list:
    return requests.get(API_ENDPOINT_URL + '''/random'''						).json()


if __name__ == "__main__":
  UpperCamelCase__    					=      random_quotes()
  pprint.pprint(response)
