
"""simple docstring"""





import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

__magic_name__       =			"bert-base-cased"
__magic_name__       =			"fp16"
__magic_name__       =			"bf16"
__magic_name__       =			[FPaa, BFaa]
@require_fsdp
@require_cuda
class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		def 	snake_case_     (     self):
				super().setUp()

				__SCREAMING_SNAKE_CASE            = dict(
				    ACCELERATE_USE_FSDP="""true"""							,  MASTER_ADDR="""localhost"""							,  MASTER_PORT="""10999"""							,  RANK="""0"""							,  LOCAL_RANK="""0"""							,  WORLD_SIZE="""1"""							,  )




		def 	snake_case_     (     self):
				from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

				for i, strategy in enumerate(lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
						__SCREAMING_SNAKE_CASE            = f"{i + 1}"
						__SCREAMING_SNAKE_CASE            = strategy
						with mockenv_context(**lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
								self.assertEqual(fsdp_plugin.sharding_strategy							,  ShardingStrategy(i + 1))




		def 	snake_case_     (     self):
				from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

				for i, prefetch_policy in enumerate(lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
						__SCREAMING_SNAKE_CASE            = prefetch_policy
						with mockenv_context(**lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
								if prefetch_policy == "NO_PREFETCH":
										self.assertIsNone(fsdp_plugin.backward_prefetch)
								else:
										self.assertEqual(fsdp_plugin.backward_prefetch							,  BackwardPrefetch(i + 1))




		def 	snake_case_     (     self):
				from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

				for i, state_dict_type in enumerate(lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
						__SCREAMING_SNAKE_CASE            = state_dict_type
						with mockenv_context(**lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
								self.assertEqual(fsdp_plugin.state_dict_type							,  StateDictType(i + 1))
								if state_dict_type == "FULL_STATE_DICT":
										self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu)
										self.assertTrue(fsdp_plugin.state_dict_config.ranka_only)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = AutoModel.from_pretrained(lowerCAmelCase__)
				for policy in FSDP_AUTO_WRAP_POLICY:
						__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
						__SCREAMING_SNAKE_CASE            = policy
						if policy == "TRANSFORMER_BASED_WRAP":
								__SCREAMING_SNAKE_CASE            = """BertLayer"""
						elif policy == "SIZE_BASED_WRAP":
								__SCREAMING_SNAKE_CASE            = """2000"""
						with mockenv_context(**lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
								fsdp_plugin.set_auto_wrap_policy(lowerCAmelCase__)
								if policy == "NO_WRAP":
										self.assertIsNone(fsdp_plugin.auto_wrap_policy)
								else:
										self.assertIsNotNone(fsdp_plugin.auto_wrap_policy)

				__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
				__SCREAMING_SNAKE_CASE            = """TRANSFORMER_BASED_WRAP"""
				__SCREAMING_SNAKE_CASE            = """T5Layer"""
				with mockenv_context(**lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
						with self.assertRaises(lowerCAmelCase__) as cm:
								fsdp_plugin.set_auto_wrap_policy(lowerCAmelCase__)
						self.assertTrue("""Could not find the transformer layer class to wrap in the model.""" in str(cm.exception))

				__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
				__SCREAMING_SNAKE_CASE            = """SIZE_BASED_WRAP"""
				__SCREAMING_SNAKE_CASE            = """0"""
				with mockenv_context(**lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
						fsdp_plugin.set_auto_wrap_policy(lowerCAmelCase__)
						self.assertIsNone(fsdp_plugin.auto_wrap_policy)




		def 	snake_case_     (     self):
				from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
				from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

				for mp_dtype in dtypes:
						__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
						__SCREAMING_SNAKE_CASE            = mp_dtype
						with mockenv_context(**lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = Accelerator()
								if mp_dtype == "fp16":
										__SCREAMING_SNAKE_CASE            = torch.floataa
								elif mp_dtype == "bf16":
										__SCREAMING_SNAKE_CASE            = torch.bfloataa
								__SCREAMING_SNAKE_CASE            = MixedPrecision(param_dtype=lowerCAmelCase__							,  reduce_dtype=lowerCAmelCase__							,  buffer_dtype=lowerCAmelCase__)
								self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy							,  lowerCAmelCase__)
								if mp_dtype == FPaa:
										self.assertTrue(isinstance(accelerator.scaler							,  lowerCAmelCase__))
								elif mp_dtype == BFaa:
										self.assertIsNone(accelerator.scaler)
								AcceleratorState._reset_state(lowerCAmelCase__)




		def 	snake_case_     (     self):
				from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

				for flag in [True, False]:
						__SCREAMING_SNAKE_CASE            = self.dist_env.copy()
						__SCREAMING_SNAKE_CASE            = str(lowerCAmelCase__).lower()
						with mockenv_context(**lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = FullyShardedDataParallelPlugin()
								self.assertEqual(fsdp_plugin.cpu_offload							,  CPUOffload(offload_params=lowerCAmelCase__))



@require_fsdp
@require_multi_gpu
@slow
class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		def 	snake_case_     (     self):
				super().setUp()
				__SCREAMING_SNAKE_CASE            = 0.82
				__SCREAMING_SNAKE_CASE            = [
				    """fsdp_shard_grad_op_transformer_based_wrap""",
				    """fsdp_full_shard_transformer_based_wrap""",
				]
				__SCREAMING_SNAKE_CASE            = {
				    """multi_gpu_fp16""": 3_2_0_0,
				    """fsdp_shard_grad_op_transformer_based_wrap_fp16""": 2_0_0_0,
				    """fsdp_full_shard_transformer_based_wrap_fp16""": 1_9_0_0,
				    # Disabling below test as it overwhelms the RAM memory usage
				    # on CI self-hosted runner leading to tests getting killed.
				    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
				}
				__SCREAMING_SNAKE_CASE            = 1_6_0
				__SCREAMING_SNAKE_CASE            = 1_6_0

				__SCREAMING_SNAKE_CASE            = inspect.getfile(accelerate.test_utils)
				__SCREAMING_SNAKE_CASE            = os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["""scripts""", """external_deps"""])




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = os.path.join(self.test_scripts_folder							,  """test_performance.py""")
				__SCREAMING_SNAKE_CASE            = ["""accelerate""", """launch""", """--num_processes=2""", """--num_machines=1""", """--machine_rank=0""", """--use_fsdp"""]
				for config in self.performance_configs:
						__SCREAMING_SNAKE_CASE            = cmd.copy()
						for i, strategy in enumerate(lowerCAmelCase__):
								if strategy.lower() in config:
										cmd_config.append(f"--fsdp_sharding_strategy={i+1}")
										break

						if "fp32" in config:
								cmd_config.append("""--mixed_precision=no""")
						else:
								cmd_config.append("""--mixed_precision=fp16""")

						if "cpu_offload" in config:
								cmd_config.append("""--fsdp_offload_params=True""")

						for policy in FSDP_AUTO_WRAP_POLICY:
								if policy.lower() in config:
										cmd_config.append(f"--fsdp_auto_wrap_policy={policy}")
										break

						if policy == "TRANSFORMER_BASED_WRAP":
								cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer""")
						elif policy == "SIZE_BASED_WRAP":
								cmd_config.append("""--fsdp_min_num_params=2000""")

						cmd_config.extend(
						    [
						        self.test_file_path,
						        f"--output_dir={self.tmpdir}",
						        f"--performance_lower_bound={self.performance_lower_bound}",
						    ])
						with patch_environment(omp_num_threads=1):
								execute_subprocess_async(lowerCAmelCase__							,  env=os.environ.copy())




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = os.path.join(self.test_scripts_folder							,  """test_checkpointing.py""")
				__SCREAMING_SNAKE_CASE            = [
				    """accelerate""",
				    """launch""",
				    """--num_processes=2""",
				    """--num_machines=1""",
				    """--machine_rank=0""",
				    """--use_fsdp""",
				    """--mixed_precision=fp16""",
				    """--fsdp_transformer_layer_cls_to_wrap=BertLayer""",
				]

				for i, strategy in enumerate(lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = cmd.copy()
						cmd_config.append(f"--fsdp_sharding_strategy={i+1}")
						if strategy != "FULL_SHARD":
								continue
						__SCREAMING_SNAKE_CASE            = len(lowerCAmelCase__)
						for state_dict_type in FSDP_STATE_DICT_TYPE:
								__SCREAMING_SNAKE_CASE            = cmd_config[:state_dict_config_index]
								cmd_config.append(f"--fsdp_state_dict_type={state_dict_type}")
								cmd_config.extend(
								    [
								        self.test_file_path,
								        f"--output_dir={self.tmpdir}",
								        """--partial_train_epoch=1""",
								    ])
								with patch_environment(omp_num_threads=1):
										execute_subprocess_async(lowerCAmelCase__							,  env=os.environ.copy())

								__SCREAMING_SNAKE_CASE            = cmd_config[:-1]
								__SCREAMING_SNAKE_CASE            = os.path.join(self.tmpdir							,  """epoch_0""")
								cmd_config.extend(
								    [
								        f"--resume_from_checkpoint={resume_from_checkpoint}",
								    ])
								with patch_environment(omp_num_threads=1):
										execute_subprocess_async(lowerCAmelCase__							,  env=os.environ.copy())




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = os.path.join(self.test_scripts_folder							,  """test_peak_memory_usage.py""")
				__SCREAMING_SNAKE_CASE            = [
				    """accelerate""",
				    """launch""",
				    """--num_processes=2""",
				    """--num_machines=1""",
				    """--machine_rank=0""",
				]
				for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
						__SCREAMING_SNAKE_CASE            = cmd.copy()
						if "fp16" in spec:
								cmd_config.extend(["""--mixed_precision=fp16"""])
						else:
								cmd_config.extend(["""--mixed_precision=no"""])

						if "multi_gpu" in spec:
								continue
						else:
								cmd_config.extend(["""--use_fsdp"""])
								for i, strategy in enumerate(lowerCAmelCase__):
										if strategy.lower() in spec:
												cmd_config.append(f"--fsdp_sharding_strategy={i+1}")
												break

								if "cpu_offload" in spec:
										cmd_config.append("""--fsdp_offload_params=True""")

								for policy in FSDP_AUTO_WRAP_POLICY:
										if policy.lower() in spec:
												cmd_config.append(f"--fsdp_auto_wrap_policy={policy}")
												break

								if policy == "TRANSFORMER_BASED_WRAP":
										cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer""")
								elif policy == "SIZE_BASED_WRAP":
										cmd_config.append("""--fsdp_min_num_params=2000""")

						cmd_config.extend(
						    [
						        self.test_file_path,
						        f"--output_dir={self.tmpdir}",
						        f"--peak_memory_upper_bound={peak_mem_upper_bound}",
						        f"--n_train={self.n_train}",
						        f"--n_val={self.n_val}",
						    ])
						with patch_environment(omp_num_threads=1):
								execute_subprocess_async(lowerCAmelCase__							,  env=os.environ.copy())











import math
from typing import Any, Callable, List, Optional, Tuple, Union

import numpy as np
import torch

from ...models import TaFilmDecoder
from ...schedulers import DDPMScheduler
from ...utils import is_onnx_available, logging, randn_tensor


if is_onnx_available():
  from ..onnx_utils import OnnxRuntimeModel

from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline
from .continous_encoder import SpectrogramContEncoder
from .notes_encoder import SpectrogramNotesEncoder


UpperCAmelCase_     							= logging.get_logger(__name__)  # pylint: disable=invalid-name

UpperCAmelCase_     							= 256






class     lowercase__       (	__lowerCamelCase				):

   '''simple docstring'''


   a						:							List[str]   		=		["melgan"]




   def __init__(    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		) ->      None:






       """simple docstring"""
       super().__init__()

       # From MELGAN
       UpperCamelCase__							:				Optional[int]          =      math.log(1E-5							)  # Matches MelGAN training.
       UpperCamelCase__							:				int          =      4.0  # Largest value for most examples
       UpperCamelCase__							:				Optional[int]          =      128

       self.register_modules(
           notes_encoder=__magic_name__,		continuous_encoder=__magic_name__,		decoder=__magic_name__,		scheduler=__magic_name__,		melgan=__magic_name__,		)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__=(-1.0, 1.0),		__magic_name__=False							) ->      Any:






       """simple docstring"""
       UpperCamelCase__   ,UpperCamelCase__							:				str          =      output_range
       if clip:
           UpperCamelCase__							:				Union[str, Any]          =      torch.clip(__magic_name__,		self.min_value,		self.max_value							)
       # Scale to [0, 1].
       UpperCamelCase__							:				List[str]          =      (features - self.min_value) / (self.max_value - self.min_value)
       # Scale to [min_out, max_out].
       return zero_one * (max_out - min_out) + min_out




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__=(-1.0, 1.0),		__magic_name__=False							) ->      Optional[int]:






       """simple docstring"""
       UpperCamelCase__   ,UpperCamelCase__							:				List[str]          =      input_range
       UpperCamelCase__							:				Any          =      torch.clip(__magic_name__,		__magic_name__,		__magic_name__							) if clip else outputs
       # Scale to [0, 1].
       UpperCamelCase__							:				Any          =      (outputs - min_out) / (max_out - min_out)
       # Scale to [self.min_value, self.max_value].
       return zero_one * (self.max_value - self.min_value) + self.min_value




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				Union[str, Any]          =      input_tokens > 0
       UpperCamelCase__   ,UpperCamelCase__							:				Any          =      self.notes_encoder(
           encoder_input_tokens=__magic_name__,		encoder_inputs_mask=__magic_name__							)

       UpperCamelCase__   ,UpperCamelCase__							:				Optional[Any]          =      self.continuous_encoder(
           encoder_inputs=__magic_name__,		encoder_inputs_mask=__magic_name__							)

       return [(tokens_encoded, tokens_mask), (continuous_encoded, continuous_mask)]




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__							) ->      str:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      noise_time
       if not torch.is_tensor(__magic_name__							):
           UpperCamelCase__							:				Tuple          =      torch.tensor([timesteps],		dtype=torch.long,		device=input_tokens.device							)
       elif torch.is_tensor(__magic_name__							) and len(timesteps.shape							) == 0:
           UpperCamelCase__							:				Union[str, Any]          =      timesteps[None].to(input_tokens.device							)

       # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
       UpperCamelCase__							:				Dict          =      timesteps * torch.ones(input_tokens.shape[0],		dtype=timesteps.dtype,		device=timesteps.device							)

       UpperCamelCase__							:				List[str]          =      self.decoder(
           encodings_and_masks=__magic_name__,		decoder_input_tokens=__magic_name__,		decoder_noise_time=__magic_name__							)
       return logits




   @torch.no_grad()
   def __call__(    self,		__magic_name__,		__magic_name__ = None,		__magic_name__ = 100,		__magic_name__ = True,		__magic_name__ = "numpy",		__magic_name__ = None,		__magic_name__ = 1,		) ->      Union[AudioPipelineOutput, Tuple]:






       """simple docstring"""
       if (callback_steps is None) or (
           callback_steps is not None and (not isinstance(__magic_name__,		__magic_name__							) or callback_steps <= 0)
       ):
           raise ValueError(
               f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
               f" {type(__magic_name__							)}."							)

       UpperCamelCase__							:				Dict          =      np.zeros([1, TARGET_FEATURE_LENGTH, self.n_dims],		dtype=np.floataa							)
       UpperCamelCase__							:				Tuple          =      np.zeros([1, 0, self.n_dims],		np.floataa							)
       UpperCamelCase__							:				List[Any]          =      torch.ones((1, TARGET_FEATURE_LENGTH),		dtype=__magic_name__,		device=self.device							)

       for i, encoder_input_tokens in enumerate(__magic_name__							):
           if i == 0:
               UpperCamelCase__							:				str          =      torch.from_numpy(pred_mel[:1].copy()							).to(
                   device=self.device,		dtype=self.decoder.dtype							)
               # The first chunk has no previous context.
               UpperCamelCase__							:				Any          =      torch.zeros((1, TARGET_FEATURE_LENGTH),		dtype=__magic_name__,		device=self.device							)
           else:
               # The full song pipeline does not feed in a context feature, so the mask
               # will be all 0s after the feature converter. Because we know we're
               # feeding in a full context chunk from the previous prediction, set it
               # to all 1s.
               UpperCamelCase__							:				List[str]          =      ones

           UpperCamelCase__							:				int          =      self.scale_features(
               __magic_name__,		output_range=[-1.0, 1.0],		clip=__magic_name__							)

           UpperCamelCase__							:				Union[str, Any]          =      self.encode(
               input_tokens=torch.IntTensor([encoder_input_tokens]							).to(device=self.device							),		continuous_inputs=__magic_name__,		continuous_mask=__magic_name__,		)

           # Sample encoder_continuous_inputs shaped gaussian noise to begin loop
           UpperCamelCase__							:				Optional[int]          =      randn_tensor(
               shape=encoder_continuous_inputs.shape,		generator=__magic_name__,		device=self.device,		dtype=self.decoder.dtype,		)

           # set step values
           self.scheduler.set_timesteps(__magic_name__							)

           # Denoising diffusion loop
           for j, t in enumerate(self.progress_bar(self.scheduler.timesteps							)							):
               UpperCamelCase__							:				Union[str, Any]          =      self.decode(
                   encodings_and_masks=__magic_name__,		input_tokens=__magic_name__,		noise_time=t / self.scheduler.config.num_train_timesteps,		)

               # Compute previous output: x_t -> x_t-1
               UpperCamelCase__							:				List[Any]          =      self.scheduler.step(__magic_name__,		__magic_name__,		__magic_name__,		generator=__magic_name__							).prev_sample

           UpperCamelCase__							:				List[Any]          =      self.scale_to_features(__magic_name__,		input_range=[-1.0, 1.0]							)
           UpperCamelCase__							:				List[Any]          =      mel[:1]
           UpperCamelCase__							:				int          =      mel.cpu().float().numpy()

           UpperCamelCase__							:				Union[str, Any]          =      np.concatenate([full_pred_mel, pred_mel[:1]],		axis=1							)

           # call the callback, if provided
           if callback is not None and i % callback_steps == 0:
               callback(__magic_name__,		__magic_name__							)

           logger.info('''Generated segment''',		__magic_name__							)

       if output_type == "numpy" and not is_onnx_available():
           raise ValueError(
               '''Cannot return output in \'np\' format if ONNX is not available. Make sure to have ONNX installed or set \'output_type\' to \'mel\'.'''							)
       elif output_type == "numpy" and self.melgan is None:
           raise ValueError(
               '''Cannot return output in \'np\' format if melgan component is not defined. Make sure to define `self.melgan` or set \'output_type\' to \'mel\'.'''							)

       if output_type == "numpy":
           UpperCamelCase__							:				Optional[int]          =      self.melgan(input_features=full_pred_mel.astype(np.floataa							)							)
       else:
           UpperCamelCase__							:				Any          =      full_pred_mel

       if not return_dict:
           return (output,)

       return AudioPipelineOutput(audios=__magic_name__							)

