


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase											=       logging.get_logger(__name__)

lowerCamelCase											=       {}




class A  (   UpperCamelCase_     ):
      UpperCamelCase__							:  int      ='llama'
      UpperCamelCase__							:  List[Any]      =['past_key_values']






      def __init__(					self						:     str			,   lowercase_						:     int=3_2000			,   lowercase_						:     Optional[int]=4096			,   lowercase_						:     List[str]=1_1008			,   lowercase_						:     Dict=32			,   lowercase_						:     List[str]=32			,   lowercase_						:     Any=None			,   lowercase_						:     Union[str, Any]="silu"			,   lowercase_						:     Tuple=2048			,   lowercase_						:     Optional[int]=0.02			,   lowercase_						:     List[Any]=1E-6			,   lowercase_						:     int=True			,   lowercase_						:     Union[str, Any]=0			,   lowercase_						:     Dict=1			,   lowercase_						:     Dict=2			,   lowercase_						:     int=1			,   lowercase_						:     Any=False			,   lowercase_						:     Dict=None			,   **lowercase_						:     Union[str, Any]			,   )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		int		=vocab_size
             _lowerCamelCase       :		List[str]		=max_position_embeddings
             _lowerCamelCase       :		List[str]		=hidden_size
             _lowerCamelCase       :		Optional[Any]		=intermediate_size
             _lowerCamelCase       :		Dict		=num_hidden_layers
             _lowerCamelCase       :		Optional[int]		=num_attention_heads

             # for backward compatibility
             if num_key_value_heads is None:
                    _lowerCamelCase       :		List[str]		=num_attention_heads

             _lowerCamelCase       :		List[Any]		=num_key_value_heads
             _lowerCamelCase       :		Optional[int]		=hidden_act
             _lowerCamelCase       :		int		=initializer_range
             _lowerCamelCase       :		Union[str, Any]		=rms_norm_eps
             _lowerCamelCase       :		Tuple		=pretraining_tp
             _lowerCamelCase       :		int		=use_cache
             _lowerCamelCase       :		List[str]		=rope_scaling
             self._rope_scaling_validation()

             super().__init__(
                 pad_token_id=lowercase_			,   bos_token_id=lowercase_			,   eos_token_id=lowercase_			,   tie_word_embeddings=lowercase_			,   **lowercase_			,   )






      def        lowerCamelCase       (					self						:     List[str] )					->						Any:
             """simple docstring"""




             if self.rope_scaling is None:
                    return

             if not isinstance(self.rope_scaling			,   lowercase_ ) or len(self.rope_scaling ) != 2:
                    raise ValueError(
                        '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                        F'''got {self.rope_scaling}''' )
             _lowerCamelCase       :		Union[str, Any]		=self.rope_scaling.get('type'			,   lowercase_ )
             _lowerCamelCase       :		Optional[Any]		=self.rope_scaling.get('factor'			,   lowercase_ )
             if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
                    raise ValueError(
                        F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''' )
             if rope_scaling_factor is None or not isinstance(lowercase_			,   lowercase_ ) or rope_scaling_factor <= 1.0:
                    raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''' )






import tempfile

import torch

from diffusers import (
    DEISMultistepScheduler,
    DPMSolverMultistepScheduler,
    DPMSolverSinglestepScheduler,
    UniPCMultistepScheduler,
)

from .test_schedulers import SchedulerCommonTest




class A  (   UpperCamelCase_     ):
      UpperCamelCase__							:  Any      =(DPMSolverSinglestepScheduler,)
      UpperCamelCase__							:  Tuple      =(('num_inference_steps', 25),)






      def        lowerCamelCase       (					self						:     Optional[Any]			,   **lowercase_						:     List[Any] )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase       :		Union[str, Any]		={
                 'num_train_timesteps': 1000,
                 'beta_start': 0.0001,
                 'beta_end': 0.02,
                 'beta_schedule': 'linear',
                 'solver_order': 2,
                 'prediction_type': 'epsilon',
                 'thresholding': False,
                 'sample_max_value': 1.0,
                 'algorithm_type': 'dpmsolver++',
                 'solver_type': 'midpoint',
                 'lambda_min_clipped': -float('inf' ),
                 'variance_type': None,
             }

             config.update(**lowercase_ )
             return config






      def        lowerCamelCase       (					self						:     Optional[int]			,   lowercase_						:     Optional[Any]=0			,   **lowercase_						:     Any )					->						List[Any]:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=dict(self.forward_default_kwargs )
             _lowerCamelCase       :		Optional[Any]		=kwargs.pop('num_inference_steps'			,   lowercase_ )
             _lowerCamelCase       :		Optional[int]		=self.dummy_sample
             _lowerCamelCase       :		Dict		=0.1 * sample
             _lowerCamelCase       :		Optional[int]		=[residual + 0.2, residual + 0.15, residual + 0.10]

             for scheduler_class in self.scheduler_classes:
                    _lowerCamelCase       :		int		=self.get_scheduler_config(**lowercase_ )
                    _lowerCamelCase       :		str		=scheduler_class(**lowercase_ )
                    scheduler.set_timesteps(lowercase_ )
                    # copy over dummy past residuals
                    _lowerCamelCase       :		int		=dummy_past_residuals[: scheduler.config.solver_order]

                    with tempfile.TemporaryDirectory() as tmpdirname:
                           scheduler.save_config(lowercase_ )
                           _lowerCamelCase       :		Dict		=scheduler_class.from_pretrained(lowercase_ )
                           new_scheduler.set_timesteps(lowercase_ )
                           # copy over dummy past residuals
                           _lowerCamelCase       :		str		=dummy_past_residuals[: new_scheduler.config.solver_order]

                    _lowerCamelCase	,     _lowerCamelCase       :		Any		=sample, sample
                    for t in range(lowercase_			,   time_step + scheduler.config.solver_order + 1 ):
                           _lowerCamelCase       :		List[str]		=scheduler.step(lowercase_			,   lowercase_			,   lowercase_			,   **lowercase_ ).prev_sample
                           _lowerCamelCase       :		Optional[Any]		=new_scheduler.step(lowercase_			,   lowercase_			,   lowercase_			,   **lowercase_ ).prev_sample

                           assert torch.sum(torch.abs(output - new_output ) ) < 1E-5, "Scheduler outputs are not identical"






      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[int]:
             """simple docstring"""




             pass






      def        lowerCamelCase       (					self						:     str			,   lowercase_						:     str=0			,   **lowercase_						:     List[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		int		=dict(self.forward_default_kwargs )
             _lowerCamelCase       :		Dict		=kwargs.pop('num_inference_steps'			,   lowercase_ )
             _lowerCamelCase       :		Optional[int]		=self.dummy_sample
             _lowerCamelCase       :		Any		=0.1 * sample
             _lowerCamelCase       :		int		=[residual + 0.2, residual + 0.15, residual + 0.10]

             for scheduler_class in self.scheduler_classes:
                    _lowerCamelCase       :		Optional[int]		=self.get_scheduler_config()
                    _lowerCamelCase       :		Any		=scheduler_class(**lowercase_ )
                    scheduler.set_timesteps(lowercase_ )

                    # copy over dummy past residuals (must be after setting timesteps)
                    _lowerCamelCase       :		Dict		=dummy_past_residuals[: scheduler.config.solver_order]

                    with tempfile.TemporaryDirectory() as tmpdirname:
                           scheduler.save_config(lowercase_ )
                           _lowerCamelCase       :		Optional[int]		=scheduler_class.from_pretrained(lowercase_ )
                           # copy over dummy past residuals
                           new_scheduler.set_timesteps(lowercase_ )

                           # copy over dummy past residual (must be after setting timesteps)
                           _lowerCamelCase       :		str		=dummy_past_residuals[: new_scheduler.config.solver_order]

                    _lowerCamelCase       :		Dict		=scheduler.step(lowercase_			,   lowercase_			,   lowercase_			,   **lowercase_ ).prev_sample
                    _lowerCamelCase       :		Dict		=new_scheduler.step(lowercase_			,   lowercase_			,   lowercase_			,   **lowercase_ ).prev_sample

                    assert torch.sum(torch.abs(output - new_output ) ) < 1E-5, "Scheduler outputs are not identical"






      def        lowerCamelCase       (					self						:     Any			,   lowercase_						:     Union[str, Any]=None			,   **lowercase_						:     List[str] )					->						str:
             """simple docstring"""




             if scheduler is None:
                    _lowerCamelCase       :		Tuple		=self.scheduler_classes[0]
                    _lowerCamelCase       :		Optional[int]		=self.get_scheduler_config(**lowercase_ )
                    _lowerCamelCase       :		Union[str, Any]		=scheduler_class(**lowercase_ )

             _lowerCamelCase       :		List[Any]		=self.scheduler_classes[0]
             _lowerCamelCase       :		Optional[Any]		=self.get_scheduler_config(**lowercase_ )
             _lowerCamelCase       :		List[Any]		=scheduler_class(**lowercase_ )

             _lowerCamelCase       :		str		=10
             _lowerCamelCase       :		Union[str, Any]		=self.dummy_model()
             _lowerCamelCase       :		Dict		=self.dummy_sample_deter
             scheduler.set_timesteps(lowercase_ )

             for i, t in enumerate(scheduler.timesteps ):
                    _lowerCamelCase       :		List[str]		=model(lowercase_			,   lowercase_ )
                    _lowerCamelCase       :		Optional[int]		=scheduler.step(lowercase_			,   lowercase_			,   lowercase_ ).prev_sample

             return sample






      def        lowerCamelCase       (					self						:     Dict )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		Any		=DPMSolverSinglestepScheduler(**self.get_scheduler_config() )
             _lowerCamelCase       :		int		=50
             _lowerCamelCase       :		Optional[int]		=self.dummy_model()
             _lowerCamelCase       :		Tuple		=self.dummy_sample_deter
             scheduler.set_timesteps(lowercase_ )

             # make sure that the first t is uneven
             for i, t in enumerate(scheduler.timesteps[3:] ):
                    _lowerCamelCase       :		Any		=model(lowercase_			,   lowercase_ )
                    _lowerCamelCase       :		List[str]		=scheduler.step(lowercase_			,   lowercase_			,   lowercase_ ).prev_sample

             _lowerCamelCase       :		int		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.2574 ) < 1E-3






      def        lowerCamelCase       (					self						:     Any )					->						List[str]:
             """simple docstring"""




             for timesteps in [25, 50, 100, 999, 1000]:
                    self.check_over_configs(num_train_timesteps=lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		=DPMSolverSinglestepScheduler(**self.get_scheduler_config() )
             _lowerCamelCase       :		Optional[Any]		=self.full_loop(scheduler=lowercase_ )
             _lowerCamelCase       :		Tuple		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.2791 ) < 1E-3

             _lowerCamelCase       :		Dict		=DEISMultistepScheduler.from_config(scheduler.config )
             _lowerCamelCase       :		Optional[int]		=DPMSolverMultistepScheduler.from_config(scheduler.config )
             _lowerCamelCase       :		Union[str, Any]		=UniPCMultistepScheduler.from_config(scheduler.config )
             _lowerCamelCase       :		List[str]		=DPMSolverSinglestepScheduler.from_config(scheduler.config )

             _lowerCamelCase       :		List[Any]		=self.full_loop(scheduler=lowercase_ )
             _lowerCamelCase       :		List[str]		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.2791 ) < 1E-3






      def        lowerCamelCase       (					self						:     str )					->						Dict:
             """simple docstring"""




             self.check_over_configs(thresholding=lowercase_ )
             for order in [1, 2, 3]:
                    for solver_type in ["midpoint", "heun"]:
                           for threshold in [0.5, 1.0, 2.0]:
                                  for prediction_type in ["epsilon", "sample"]:
                                         self.check_over_configs(
                                             thresholding=lowercase_			,   prediction_type=lowercase_			,   sample_max_value=lowercase_			,   algorithm_type='dpmsolver++'			,   solver_order=lowercase_			,   solver_type=lowercase_			,   )






      def        lowerCamelCase       (					self						:     Any )					->						Any:
             """simple docstring"""




             for prediction_type in ["epsilon", "v_prediction"]:
                    self.check_over_configs(prediction_type=lowercase_ )






      def        lowerCamelCase       (					self						:     List[str] )					->						Optional[int]:
             """simple docstring"""




             for algorithm_type in ["dpmsolver", "dpmsolver++"]:
                    for solver_type in ["midpoint", "heun"]:
                           for order in [1, 2, 3]:
                                  for prediction_type in ["epsilon", "sample"]:
                                         self.check_over_configs(
                                             solver_order=lowercase_			,   solver_type=lowercase_			,   prediction_type=lowercase_			,   algorithm_type=lowercase_			,   )
                                         _lowerCamelCase       :		Any		=self.full_loop(
                                             solver_order=lowercase_			,   solver_type=lowercase_			,   prediction_type=lowercase_			,   algorithm_type=lowercase_			,   )
                                         assert not torch.isnan(lowercase_ ).any(), "Samples have nan numbers"






      def        lowerCamelCase       (					self						:     List[Any] )					->						List[Any]:
             """simple docstring"""




             self.check_over_configs(lower_order_final=lowercase_ )
             self.check_over_configs(lower_order_final=lowercase_ )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						int:
             """simple docstring"""




             self.check_over_configs(lambda_min_clipped=-float('inf' ) )
             self.check_over_configs(lambda_min_clipped=-5.1 )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Any:
             """simple docstring"""




             self.check_over_configs(variance_type=lowercase_ )
             self.check_over_configs(variance_type='learned_range' )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Any:
             """simple docstring"""




             for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]:
                    self.check_over_forward(num_inference_steps=lowercase_			,   time_step=0 )






      def        lowerCamelCase       (					self						:     str )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Dict		=self.full_loop()
             _lowerCamelCase       :		Dict		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.2791 ) < 1E-3






      def        lowerCamelCase       (					self						:     Any )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		str		=self.full_loop(use_karras_sigmas=lowercase_ )
             _lowerCamelCase       :		Optional[Any]		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.2248 ) < 1E-3






      def        lowerCamelCase       (					self						:     Dict )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Any		=self.full_loop(prediction_type='v_prediction' )
             _lowerCamelCase       :		Dict		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.1453 ) < 1E-3






      def        lowerCamelCase       (					self						:     Any )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		str		=self.full_loop(prediction_type='v_prediction'			,   use_karras_sigmas=lowercase_ )
             _lowerCamelCase       :		List[str]		=torch.mean(torch.abs(lowercase_ ) )

             assert abs(result_mean.item() - 0.0649 ) < 1E-3






      def        lowerCamelCase       (					self						:     List[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=self.scheduler_classes[0]
             _lowerCamelCase       :		List[str]		=self.get_scheduler_config(thresholding=lowercase_			,   dynamic_thresholding_ratio=0 )
             _lowerCamelCase       :		List[Any]		=scheduler_class(**lowercase_ )

             _lowerCamelCase       :		Optional[Any]		=10
             _lowerCamelCase       :		Optional[int]		=self.dummy_model()
             _lowerCamelCase       :		List[Any]		=self.dummy_sample_deter.half()
             scheduler.set_timesteps(lowercase_ )

             for i, t in enumerate(scheduler.timesteps ):
                    _lowerCamelCase       :		str		=model(lowercase_			,   lowercase_ )
                    _lowerCamelCase       :		Any		=scheduler.step(lowercase_			,   lowercase_			,   lowercase_ ).prev_sample

             assert sample.dtype == torch.floataa


