


import unittest

import numpy as np
import torch

from diffusers import DDPMPipeline, DDPMScheduler, UNetaDModel
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device


enable_full_determinism()





class       _a							(       unittest.TestCase):




	@property
	def 							UpperCAmelCase__(      self		:	Dict	)-> int:
			torch.manual_seed(0	)
			lowerCAmelCase__     :    int					  =      UNetaDModel(
			    block_out_channels=(32, 64)	,      layers_per_block=2	,      sample_size=32	,      in_channels=3	,      out_channels=3	,      down_block_types=('''DownBlock2D''', '''AttnDownBlock2D''')	,      up_block_types=('''AttnUpBlock2D''', '''UpBlock2D''')	,      )
			return model




	def 							UpperCAmelCase__(      self		:	Union[str, Any]	)-> Dict:
			lowerCAmelCase__     :    Union[str, Any]					  =      '''cpu'''
			lowerCAmelCase__     :    List[Any]					  =      self.dummy_uncond_unet
			lowerCAmelCase__     :    Optional[Any]					  =      DDPMScheduler()

			lowerCAmelCase__     :    List[str]					  =      DDPMPipeline(unet=_SCREAMING_SNAKE_CASE	,      scheduler=_SCREAMING_SNAKE_CASE	)
			ddpm.to(_SCREAMING_SNAKE_CASE	)
			ddpm.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE	)

			lowerCAmelCase__     :    Optional[Any]					  =      torch.Generator(device=_SCREAMING_SNAKE_CASE	).manual_seed(0	)
			lowerCAmelCase__     :    int					  =      ddpm(generator=_SCREAMING_SNAKE_CASE	,      num_inference_steps=2	,      output_type='''numpy'''	).images

			lowerCAmelCase__     :    str					  =      torch.Generator(device=_SCREAMING_SNAKE_CASE	).manual_seed(0	)
			lowerCAmelCase__     :    Optional[int]					  =      ddpm(generator=_SCREAMING_SNAKE_CASE	,      num_inference_steps=2	,      output_type='''numpy'''	,      return_dict=_SCREAMING_SNAKE_CASE	)[0]

			lowerCAmelCase__     :    Tuple					  =      image[0, -3:, -3:, -1]
			lowerCAmelCase__     :    List[str]					  =      image_from_tuple[0, -3:, -3:, -1]

			assert image.shape == (1, 32, 32, 3)
			lowerCAmelCase__     :    int					  =      np.array(
			    [9.956E-01, 5.785E-01, 4.675E-01, 9.930E-01, 0.0, 1.000, 1.199E-03, 2.648E-04, 5.101E-04]	)

			assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2
			assert np.abs(image_from_tuple_slice.flatten() - expected_slice	).max() < 1E-2




	def 							UpperCAmelCase__(      self		:	List[str]	)-> Any:
			lowerCAmelCase__     :    int					  =      self.dummy_uncond_unet
			lowerCAmelCase__     :    List[Any]					  =      DDPMScheduler(prediction_type='''sample'''	)

			lowerCAmelCase__     :    int					  =      DDPMPipeline(unet=_SCREAMING_SNAKE_CASE	,      scheduler=_SCREAMING_SNAKE_CASE	)
			ddpm.to(_SCREAMING_SNAKE_CASE	)
			ddpm.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE	)

			lowerCAmelCase__     :    Optional[int]					  =      torch.manual_seed(0	)
			lowerCAmelCase__     :    List[Any]					  =      ddpm(generator=_SCREAMING_SNAKE_CASE	,      num_inference_steps=2	,      output_type='''numpy'''	).images

			lowerCAmelCase__     :    Optional[Any]					  =      torch.manual_seed(0	)
			lowerCAmelCase__     :    Union[str, Any]					  =      ddpm(generator=_SCREAMING_SNAKE_CASE	,      num_inference_steps=2	,      output_type='''numpy'''	)[0]

			lowerCAmelCase__     :    str					  =      image[0, -3:, -3:, -1]
			lowerCAmelCase__     :    Optional[Any]					  =      image_eps[0, -3:, -3:, -1]

			assert image.shape == (1, 32, 32, 3)
			lowerCAmelCase__     :    Any					  =      1E-2 if torch_device != '''mps''' else 3E-2
			assert np.abs(image_slice.flatten() - image_eps_slice.flatten()	).max() < tolerance





@slow
@require_torch_gpu
class       _a							(       unittest.TestCase):




	def 							UpperCAmelCase__(      self		:	Any	)-> List[Any]:
			lowerCAmelCase__     :    str					  =      '''google/ddpm-cifar10-32'''

			lowerCAmelCase__     :    List[Any]					  =      UNetaDModel.from_pretrained(_SCREAMING_SNAKE_CASE	)
			lowerCAmelCase__     :    Any					  =      DDPMScheduler.from_pretrained(_SCREAMING_SNAKE_CASE	)

			lowerCAmelCase__     :    List[str]					  =      DDPMPipeline(unet=_SCREAMING_SNAKE_CASE	,      scheduler=_SCREAMING_SNAKE_CASE	)
			ddpm.to(_SCREAMING_SNAKE_CASE	)
			ddpm.set_progress_bar_config(disable=_SCREAMING_SNAKE_CASE	)

			lowerCAmelCase__     :    List[Any]					  =      torch.manual_seed(0	)
			lowerCAmelCase__     :    Optional[int]					  =      ddpm(generator=_SCREAMING_SNAKE_CASE	,      output_type='''numpy'''	).images

			lowerCAmelCase__     :    List[Any]					  =      image[0, -3:, -3:, -1]

			assert image.shape == (1, 32, 32, 3)
			lowerCAmelCase__     :    List[str]					  =      np.array([0.4200, 0.3588, 0.1939, 0.3847, 0.3382, 0.2647, 0.4155, 0.3582, 0.3385]	)
			assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2








import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
		import torch

		from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
		import pyanvml.pyanvml as nvml


lowerCamelCase		=	logging.get_logger(__name__)





class       _a							(       _lowercase):
	_a						:						PyTorchBenchmarkArguments
	_a						:						PretrainedConfig
	_a						:						str   					=							"PyTorch"




	@property
	def 							UpperCAmelCase__(      self		:	Dict	)-> Any:
			return torch.__version__




	def 							UpperCAmelCase__(      self		:	List[Any]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> float:
			lowerCAmelCase__     :    Optional[Any]					  =      self._prepare_inference_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_speed(_inference	)




	def 							UpperCAmelCase__(      self		:	List[str]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> [Memory, Optional[MemorySummary]]:
			lowerCAmelCase__     :    List[Any]					  =      self._prepare_inference_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_memory(_inference	)




	def 							UpperCAmelCase__(      self		:	Tuple	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> float:
			lowerCAmelCase__     :    Any					  =      self._prepare_train_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_speed(_train	)




	def 							UpperCAmelCase__(      self		:	Optional[Any]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> [Memory, Optional[MemorySummary]]:
			lowerCAmelCase__     :    Tuple					  =      self._prepare_train_func(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			return self._measure_memory(_train	)




	def 							UpperCAmelCase__(      self		:	int	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> Callable[[], None]:
			lowerCAmelCase__     :    int					  =      self.config_dict[model_name]

			if self.args.torchscript:
					lowerCAmelCase__     :    List[Any]					  =      True

			lowerCAmelCase__     :    List[Any]					  =      (
			    hasattr(_SCREAMING_SNAKE_CASE	,      '''architectures'''	)
			    and isinstance(config.architectures	,      _SCREAMING_SNAKE_CASE	)
			    and len(config.architectures	) > 0
			)
			if not self.args.only_pretrain_model and has_model_class_in_config:
					try:
							lowerCAmelCase__     :    Dict					  =      config.architectures[0]
							lowerCAmelCase__     :    Optional[Any]					  =      __import__('''transformers'''	,      fromlist=[model_class]	)
							lowerCAmelCase__     :    Any					  =      getattr(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__     :    List[Any]					  =      model_cls(_SCREAMING_SNAKE_CASE	)
					except ImportError:
							raise ImportError(
							    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
							    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''	)
			else:
					lowerCAmelCase__     :    Union[str, Any]					  =      MODEL_MAPPING[config.__class__](_SCREAMING_SNAKE_CASE	)

			model.eval()
			model.to(self.args.device	)

			# encoder-decoder has vocab size saved differently
			lowerCAmelCase__     :    str					  =      config.vocab_size if hasattr(_SCREAMING_SNAKE_CASE	,      '''vocab_size'''	) else config.encoder.vocab_size
			lowerCAmelCase__     :    Dict					  =      torch.randint(_SCREAMING_SNAKE_CASE	,      (batch_size, sequence_length)	,      dtype=torch.long	,      device=self.args.device	)

			if self.args.fpaa:
					logger.info('''Running training in Mixed Precision...'''	)
					if not self.args.is_gpu:
							raise ValueError('''Mixed precision is possible only for GPU.'''	)
					# amp seems to have memory leaks so that memory usage
					# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
					model.half()

			if self.args.torchscript:
					with torch.no_grad():
							lowerCAmelCase__     :    int					  =      torch.jit.trace(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			else:
					lowerCAmelCase__     :    Optional[int]					  =      model

			def encoder_decoder_forward():
					with torch.no_grad():
							lowerCAmelCase__     :    str					  =      inference_model(_SCREAMING_SNAKE_CASE	,      decoder_input_ids=_SCREAMING_SNAKE_CASE	)
					return outputs

			def encoder_forward():
					with torch.no_grad():
							lowerCAmelCase__     :    List[Any]					  =      inference_model(_SCREAMING_SNAKE_CASE	)
					return outputs

			lowerCAmelCase__     :    Any					  =      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
			return _forward




	def 							UpperCAmelCase__(      self		:	str	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int	)-> Callable[[], None]:
			lowerCAmelCase__     :    Optional[int]					  =      self.config_dict[model_name]

			lowerCAmelCase__     :    List[str]					  =      (
			    hasattr(_SCREAMING_SNAKE_CASE	,      '''architectures'''	)
			    and isinstance(config.architectures	,      _SCREAMING_SNAKE_CASE	)
			    and len(config.architectures	) > 0
			)
			if not self.args.only_pretrain_model and has_model_class_in_config:
					try:
							lowerCAmelCase__     :    Optional[Any]					  =      config.architectures[0]
							lowerCAmelCase__     :    List[str]					  =      __import__('''transformers'''	,      fromlist=[model_class]	)
							lowerCAmelCase__     :    str					  =      getattr(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__     :    Union[str, Any]					  =      model_cls(_SCREAMING_SNAKE_CASE	)
					except ImportError:
							raise ImportError(
							    F'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
							    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''	)
			else:
					lowerCAmelCase__     :    Any					  =      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_SCREAMING_SNAKE_CASE	)

			if self.args.torchscript:
					raise NotImplementedError('''Training for torchscript is currently not implemented'''	)
			else:
					lowerCAmelCase__     :    Union[str, Any]					  =      model

			model.train()
			model.to(self.args.device	)

			# encoder-decoder has vocab size saved differently
			lowerCAmelCase__     :    Union[str, Any]					  =      config.vocab_size if hasattr(_SCREAMING_SNAKE_CASE	,      '''vocab_size'''	) else config.encoder.vocab_size
			lowerCAmelCase__     :    List[str]					  =      torch.randint(_SCREAMING_SNAKE_CASE	,      (batch_size, sequence_length)	,      dtype=torch.long	,      device=self.args.device	)

			if self.args.fpaa:
					logger.info('''Running training in Mixed Precision...'''	)
					if not self.args.is_gpu:
							raise ValueError('''Mixed precision is possible only for GPU.'''	)

					# amp seems to have memory leaks so that memory usage
					# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
					model.half()

			def compute_loss_and_backprob_encoder():
					lowerCAmelCase__     :    Optional[Any]					  =      train_model(_SCREAMING_SNAKE_CASE	,      labels=_SCREAMING_SNAKE_CASE	)[0]
					loss.backward()
					return loss

			def compute_loss_and_backprob_encoder_decoder():
					lowerCAmelCase__     :    List[str]					  =      train_model(_SCREAMING_SNAKE_CASE	,      decoder_input_ids=_SCREAMING_SNAKE_CASE	,      labels=_SCREAMING_SNAKE_CASE	)[0]
					loss.backward()
					return loss

			lowerCAmelCase__     :    List[Any]					  =      (
			    compute_loss_and_backprob_encoder_decoder
			    if config.is_encoder_decoder
			    else compute_loss_and_backprob_encoder
			)
			return _train




	def 							UpperCAmelCase__(      self		:	Union[str, Any]	,      _SCREAMING_SNAKE_CASE		:	List[str]	)-> float:
			try:
					if self.args.is_tpu or self.args.torchscript:
							# run additional 10 times to stabilize compilation for tpu and torchscript
							logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''	)
							timeit.repeat(
							    _SCREAMING_SNAKE_CASE	,      repeat=1	,      number=5	,      )

					# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
					lowerCAmelCase__     :    Tuple					  =      timeit.repeat(
					    _SCREAMING_SNAKE_CASE	,      repeat=self.args.repeat	,      number=10	,      )

					if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
							import torch_xla.debug.metrics as met

							self.print_fn(met.metrics_report()	)

					return min(_SCREAMING_SNAKE_CASE	) / 10.0
			except RuntimeError as e:
					self.print_fn(F'Doesn\'t fit on GPU. {e}'	)
					return "N/A"




	def 							UpperCAmelCase__(      self		:	int	,      _SCREAMING_SNAKE_CASE		:	Callable[[], None]	)-> [Memory, MemorySummary]:
			try:
					if self.args.trace_memory_line_by_line:
							lowerCAmelCase__     :    int					  =      start_memory_tracing('''transformers'''	)

					if self.args.is_tpu:
							# tpu
							raise NotImplementedError(
							    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
							    ''' `--no-memory` or `args.memory=False`'''	)
					elif self.args.is_gpu:
							if not is_pyanvml_available():
									logger.warning(
									    '''py3nvml not installed, we won\'t log GPU memory usage. '''
									    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''	)
									lowerCAmelCase__     :    Union[str, Any]					  =      '''N/A'''
							else:
									logger.info(
									    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
									    ''' on the same GPU.'''	)
									# init nvml
									nvml.nvmlInit()
									func()
									lowerCAmelCase__     :    Optional[int]					  =      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx	)
									lowerCAmelCase__     :    Any					  =      nvml.nvmlDeviceGetMemoryInfo(_SCREAMING_SNAKE_CASE	)
									lowerCAmelCase__     :    int					  =      meminfo.used
									lowerCAmelCase__     :    Tuple					  =      Memory(_SCREAMING_SNAKE_CASE	)
									# shutdown nvml
									nvml.nvmlShutdown()
					else:
							# cpu
							lowerCAmelCase__     :    Union[str, Any]					  =      measure_peak_memory_cpu(_SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__     :    Union[str, Any]					  =      Memory(_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	) else memory_bytes

					if self.args.trace_memory_line_by_line:
							lowerCAmelCase__     :    int					  =      stop_memory_tracing(_SCREAMING_SNAKE_CASE	)
					else:
							lowerCAmelCase__     :    str					  =      None

					return memory, summary
			except RuntimeError as e:
					self.print_fn(F'Doesn\'t fit on GPU. {e}'	)
					return "N/A", None




