'''simple docstring'''




import unittest
from typing import Tuple

import torch

from diffusers.utils import floats_tensor, randn_tensor, torch_all_close, torch_device
from diffusers.utils.testing_utils import require_torch




@require_torch
class     a__      :






     @property
     def    SCREAMING_SNAKE_CASE__   ( self		:		Union[str, Any]    ):





         """simple docstring"""



         return self.get_dummy_input()






     @property
     def    SCREAMING_SNAKE_CASE__   ( self		:		str    ):





         """simple docstring"""



         if self.block_type == "down":
             return (4, 32, 16, 16)
         elif self.block_type == "mid":
             return (4, 32, 32, 32)
         elif self.block_type == "up":
             return (4, 32, 64, 64)

         raise ValueError(f"""'{self.block_type}' is not a supported block_type. Set it to 'up', 'mid', or 'down'."""    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Tuple  ,				a		:		Any=True  ,				a		:		Any=False  ,				a		:		List[Any]=False  ,				a		:		List[Any]=False  ,				):





         """simple docstring"""



         __lowerCamelCase		=  4
         __lowerCamelCase		=  32
         __lowerCamelCase		=  (32, 32)

         __lowerCamelCase		=  torch.manual_seed(0    )
         __lowerCamelCase		=  torch.device(a    )
         __lowerCamelCase		=  (batch_size, num_channels) + sizes
         __lowerCamelCase		=  randn_tensor(a  ,				generator=a  ,				device=a    )
         __lowerCamelCase		=  {'''hidden_states''': hidden_states}

         if include_temb:
             __lowerCamelCase		=  1_28
             __lowerCamelCase		=  randn_tensor((batch_size, temb_channels)  ,				generator=a  ,				device=a    )

         if include_res_hidden_states_tuple:
             __lowerCamelCase		=  torch.manual_seed(1    )
             __lowerCamelCase		=  (randn_tensor(a  ,				generator=a  ,				device=a    ),)

         if include_encoder_hidden_states:
             __lowerCamelCase		=  floats_tensor((batch_size, 32, 32)    ).to(a    )

         if include_skip_sample:
             __lowerCamelCase		=  randn_tensor(((batch_size, 3) + sizes)  ,				generator=a  ,				device=a    )

         return dummy_input






     def    SCREAMING_SNAKE_CASE__   ( self		:		Dict    ):





         """simple docstring"""



         __lowerCamelCase		=  {
             '''in_channels''': 32,
             '''out_channels''': 32,
             '''temb_channels''': 1_28,
         }
         if self.block_type == "up":
             __lowerCamelCase		=  32

         if self.block_type == "mid":
             init_dict.pop('''out_channels'''    )

         __lowerCamelCase		=  self.dummy_input
         return init_dict, inputs_dict






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[Any]  ,				a		:		str    ):





         """simple docstring"""



         __lowerCamelCase	,  __lowerCamelCase		=  self.prepare_init_args_and_inputs_for_common()
         __lowerCamelCase		=  self.block_class(**a    )
         unet_block.to(a    )
         unet_block.eval()

         with torch.no_grad():
             __lowerCamelCase		=  unet_block(**a    )

         if isinstance(a  ,				a    ):
             __lowerCamelCase		=  output[0]

         self.assertEqual(output.shape  ,				self.output_shape    )

         __lowerCamelCase		=  output[0, -1, -3:, -3:]
         __lowerCamelCase		=  torch.tensor(a    ).to(a    )
         assert torch_all_close(output_slice.flatten()  ,				a  ,				atol=5e-3    )






     @unittest.skipIf(torch_device == '''mps'''  ,				'''Training is not supported in mps'''    )
     def    SCREAMING_SNAKE_CASE__   ( self		:		List[str]    ):





         """simple docstring"""



         __lowerCamelCase	,  __lowerCamelCase		=  self.prepare_init_args_and_inputs_for_common()
         __lowerCamelCase		=  self.block_class(**a    )
         model.to(a    )
         model.train()
         __lowerCamelCase		=  model(**a    )

         if isinstance(a  ,				a    ):
             __lowerCamelCase		=  output[0]

         __lowerCamelCase		=  torch.device(a    )
         __lowerCamelCase		=  randn_tensor(output.shape  ,				device=a    )
         __lowerCamelCase		=  torch.nn.functional.mse_loss(a  ,				a    )
         loss.backward()

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase		:       Any									=     logging.get_logger(__name__)

__UpperCamelCase		:       Optional[int]									=     {
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}





class 					lowercase__						(   UpperCamelCase_):
						UpperCamelCase_     =		"""nllb-moe"""
						UpperCamelCase_     =		["""past_key_values"""]
						UpperCamelCase_     =		{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}


						def __init__(       self	:				List[str]    ,		UpperCamelCase__	:				List[str]=12_8112    ,		UpperCamelCase__	:				str=1024    ,		UpperCamelCase__	:				Optional[int]=12    ,		UpperCamelCase__	:				Union[str, Any]=4096    ,		UpperCamelCase__	:				Optional[Any]=16    ,		UpperCamelCase__	:				Optional[Any]=12    ,		UpperCamelCase__	:				str=4096    ,		UpperCamelCase__	:				Dict=16    ,		UpperCamelCase__	:				Any=0.05    ,		UpperCamelCase__	:				Any=0.05    ,		UpperCamelCase__	:				Optional[Any]=True    ,		UpperCamelCase__	:				Optional[Any]=True    ,		UpperCamelCase__	:				List[Any]="relu"    ,		UpperCamelCase__	:				Union[str, Any]=1024    ,		UpperCamelCase__	:				Optional[int]=0.1    ,		UpperCamelCase__	:				Any=0.1    ,		UpperCamelCase__	:				Optional[Any]=0.0    ,		UpperCamelCase__	:				str=0.02    ,		UpperCamelCase__	:				Tuple=2    ,		UpperCamelCase__	:				List[str]=True    ,		UpperCamelCase__	:				Dict=False    ,		UpperCamelCase__	:				Any="float32"    ,		UpperCamelCase__	:				List[Any]=False    ,		UpperCamelCase__	:				Tuple=128    ,		UpperCamelCase__	:				Tuple=64    ,		UpperCamelCase__	:				List[str]=4    ,		UpperCamelCase__	:				Union[str, Any]=4    ,		UpperCamelCase__	:				Dict=0.001    ,		UpperCamelCase__	:				Optional[Any]=0.001    ,		UpperCamelCase__	:				Optional[Any]="all"    ,		UpperCamelCase__	:				Tuple=False    ,		UpperCamelCase__	:				List[Any]=False    ,		UpperCamelCase__	:				List[Any]=1.0    ,		UpperCamelCase__	:				str=0.2    ,		UpperCamelCase__	:				List[Any]=1    ,		UpperCamelCase__	:				List[Any]=0    ,		UpperCamelCase__	:				Optional[int]=2    ,		UpperCamelCase__	:				Tuple=False    ,		**UpperCamelCase__	:				Union[str, Any]    ,		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    List[str]							=							vocab_size
												SCREAMING_SNAKE_CASE							:    Dict							=							max_position_embeddings
												SCREAMING_SNAKE_CASE							:    int							=							d_model
												SCREAMING_SNAKE_CASE							:    Any							=							encoder_ffn_dim
												SCREAMING_SNAKE_CASE							:    List[Any]							=							encoder_layers
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							encoder_attention_heads
												SCREAMING_SNAKE_CASE							:    List[str]							=							decoder_ffn_dim
												SCREAMING_SNAKE_CASE							:    Dict							=							decoder_layers
												SCREAMING_SNAKE_CASE							:    List[Any]							=							decoder_attention_heads
												SCREAMING_SNAKE_CASE							:    List[str]							=							dropout
												SCREAMING_SNAKE_CASE							:    Any							=							attention_dropout
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							activation_dropout
												SCREAMING_SNAKE_CASE							:    List[Any]							=							activation_function
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							init_std
												SCREAMING_SNAKE_CASE							:    int							=							encoder_layerdrop
												SCREAMING_SNAKE_CASE							:    List[Any]							=							decoder_layerdrop
												SCREAMING_SNAKE_CASE							:    Any							=							use_cache
												SCREAMING_SNAKE_CASE							:    str							=							encoder_layers
												SCREAMING_SNAKE_CASE							:    List[Any]							=							scale_embedding  # scale factor will be sqrt(d_model) if True
												SCREAMING_SNAKE_CASE							:    List[str]							=							router_z_loss_coef
												SCREAMING_SNAKE_CASE							:    List[str]							=							router_aux_loss_coef
												SCREAMING_SNAKE_CASE							:    int							=							decoder_sparse_step
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							encoder_sparse_step
												SCREAMING_SNAKE_CASE							:    List[str]							=							num_experts
												SCREAMING_SNAKE_CASE							:    int							=							expert_capacity
												SCREAMING_SNAKE_CASE							:    Any							=							router_bias
												if router_dtype not in ["float32", "float16", "bfloat16"]:
																		raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""						)
												SCREAMING_SNAKE_CASE							:    str							=							router_dtype

												SCREAMING_SNAKE_CASE							:    List[Any]							=							router_ignore_padding_tokens
												SCREAMING_SNAKE_CASE							:    int							=							batch_prioritized_routing
												SCREAMING_SNAKE_CASE							:    str							=							second_expert_policy
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							normalize_router_prob_before_dropping
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							moe_eval_capacity_token_fraction
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							moe_token_dropout
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							output_router_logits
												super().__init__(
												    pad_token_id=UpperCamelCase__    ,		bos_token_id=UpperCamelCase__    ,		eos_token_id=UpperCamelCase__    ,		is_encoder_decoder=UpperCamelCase__    ,		decoder_start_token_id=UpperCamelCase__    ,		**UpperCamelCase__    ,		)





