






import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


lowerCamelCase					    =	[nn.LayerNorm]

lowerCamelCase					    =	logging.get_logger(__name__)

lowerCamelCase					    =	version.parse(version.parse(torch.__version__).base_version)

lowerCamelCase					    =	parsed_torch_version_base >= version.parse('''2.0''')
lowerCamelCase					    =	parsed_torch_version_base >= version.parse('''1.12''')
lowerCamelCase					    =	parsed_torch_version_base >= version.parse('''1.11''')
lowerCamelCase					    =	parsed_torch_version_base < version.parse('''1.11''')
def 							UpperCAmelCase   ( _lowercase,   _lowercase,   _lowercase,   _lowercase,   self   ):


    '''simple docstring'''



    from torch import _softmax_backward_data

    if is_torch_less_than_1_11:
        return _softmax_backward_data(A_,   A_,   parent.dim,   self   )
    else:
        return _softmax_backward_data(A_,   A_,   parent.dim,   self.dtype   )
def 							UpperCAmelCase   ( _lowercase,   _lowercase,   _lowercase = 0   ):


    '''simple docstring'''



    _lowercase     			=						index.to(layer.weight.device   )
    _lowercase     			=						layer.weight.index_select(A_,   A_   ).clone().detach()
    if layer.bias is not None:
        if dim == 1:
            _lowercase     			=						layer.bias.clone().detach()
        else:
            _lowercase     			=						layer.bias[index].clone().detach()
    _lowercase     			=						list(layer.weight.size()   )
    _lowercase     			=						len(A_   )
    _lowercase     			=						nn.Linear(new_size[1],   new_size[0],   bias=layer.bias is not None   ).to(layer.weight.device   )
    _lowercase     			=						False
    new_layer.weight.copy_(W.contiguous()   )
    _lowercase     			=						True
    if layer.bias is not None:
        _lowercase     			=						False
        new_layer.bias.copy_(b.contiguous()   )
        _lowercase     			=						True
    return new_layer

class        snake_case_				(							nn.Module   ):







   """simple docstring"""



   def __init__(     self:						str ,							snake_case__:						int ,							snake_case__:						Optional[Any]						)	->		str:


       """simple docstring"""



       super().__init__()
       _lowercase     			=						nf
       _lowercase     			=						nn.Parameter(torch.empty(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_						)						)
       _lowercase     			=						nn.Parameter(torch.zeros(SCREAMING_SNAKE_CASE_						)						)
       nn.init.normal_(self.weight ,							std=0.02						)

   def       SCREAMING_SNAKE_CASE__					(     self:						List[Any] ,							snake_case__:						Optional[int]						)	->		Dict:


       """simple docstring"""



       _lowercase     			=						x.size()[:-1] + (self.nf,)
       _lowercase     			=						torch.addmm(self.bias ,							x.view(-1 ,							x.size(-1						)						) ,							self.weight						)
       _lowercase     			=						x.view(SCREAMING_SNAKE_CASE_						)
       return x






def 							UpperCAmelCase   ( _lowercase,   _lowercase,   _lowercase = 1   ):


    '''simple docstring'''



    _lowercase     			=						index.to(layer.weight.device   )
    _lowercase     			=						layer.weight.index_select(A_,   A_   ).clone().detach()
    if dim == 0:
        _lowercase     			=						layer.bias.clone().detach()
    else:
        _lowercase     			=						layer.bias[index].clone().detach()
    _lowercase     			=						list(layer.weight.size()   )
    _lowercase     			=						len(A_   )
    _lowercase     			=						ConvaD(new_size[1],   new_size[0]   ).to(layer.weight.device   )
    _lowercase     			=						False
    new_layer.weight.copy_(W.contiguous()   )
    _lowercase     			=						True
    _lowercase     			=						False
    new_layer.bias.copy_(b.contiguous()   )
    _lowercase     			=						True
    return new_layer
def 							UpperCAmelCase   ( _lowercase,   _lowercase,   _lowercase = None   ):


    '''simple docstring'''



    if isinstance(A_,   nn.Linear   ):
        return prune_linear_layer(A_,   A_,   dim=0 if dim is None else dim   )
    elif isinstance(A_,   A_   ):
        return prune_convad_layer(A_,   A_,   dim=1 if dim is None else dim   )
    else:
        raise ValueError(f'''Can\'t prune layer of class {layer.__class__}'''   )
def 							UpperCAmelCase   ( _lowercase,   _lowercase,   _lowercase,   *_lowercase   ):


    '''simple docstring'''



    assert len(A_   ) > 0, f'''{input_tensors} has to be a tuple/list of tensors'''

    # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
    _lowercase     			=						len(inspect.signature(A_   ).parameters   )
    if num_args_in_forward_chunk_fn != len(A_   ):
        raise ValueError(
            f'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(A_   )} input '''
            'tensors are given'   )

    if chunk_size > 0:
        _lowercase     			=						input_tensors[0].shape[chunk_dim]
        for input_tensor in input_tensors:
            if input_tensor.shape[chunk_dim] != tensor_shape:
                raise ValueError(
                    f'''All input tenors have to be of the same shape: {tensor_shape}, '''
                    f'''found shape {input_tensor.shape[chunk_dim]}'''   )

        if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
            raise ValueError(
                f'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
                f'''size {chunk_size}'''   )

        _lowercase     			=						input_tensors[0].shape[chunk_dim] // chunk_size

        # chunk input tensor into tuples
        _lowercase     			=						tuple(input_tensor.chunk(A_,   dim=A_   ) for input_tensor in input_tensors   )
        # apply forward fn to every tuple
        _lowercase     			=						tuple(forward_fn(*A_   ) for input_tensors_chunk in zip(*A_   )   )
        # concatenate output at same dimension
        return torch.cat(A_,   dim=A_   )

    return forward_fn(*A_   )
def 							UpperCAmelCase   ( _lowercase,   _lowercase,   _lowercase,   _lowercase   ):


    '''simple docstring'''



    _lowercase     			=						torch.ones(A_,   A_   )
    _lowercase     			=						set(A_   ) - already_pruned_heads  # Convert to set and remove already pruned heads
    for head in heads:
        # Compute how many pruned heads are before the head and move the index accordingly
        _lowercase     			=						head - sum(1 if h < head else 0 for h in already_pruned_heads   )
        _lowercase     			=						0
    _lowercase     			=						mask.view(-1   ).contiguous().eq(1   )
    _lowercase     			=						torch.arange(len(A_   )   )[mask].long()
    return heads, index
def 							UpperCAmelCase   ( *_lowercase,   _lowercase = None   ):


    '''simple docstring'''



    return torch.meshgrid(*A_,   indexing=A_   )
def 							UpperCAmelCase   ( _lowercase   ):


    '''simple docstring'''



    return tensor.device, storage_ptr(A_   ), storage_size(A_   )


'''simple docstring'''





import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


UpperCAmelCase            = logging.get_logger(__name__)

UpperCAmelCase            = {
    "facebook/musicgen-small": "https://huggingface.co/facebook/musicgen-small/resolve/main/config.json",
    # See all Musicgen models at https://huggingface.co/models?filter=musicgen
}

class     a (					__magic_name__     ):
				_snake_case		     =			'''musicgen_decoder'''
				_snake_case		     =			['''past_key_values''']




				def __init__(				self						:					Optional[int],     SCREAMING_SNAKE_CASE_						:					Dict=20_48,     SCREAMING_SNAKE_CASE_						:					List[str]=20_48,     SCREAMING_SNAKE_CASE_						:					int=24,     SCREAMING_SNAKE_CASE_						:					Dict=40_96,     SCREAMING_SNAKE_CASE_						:					str=16,     SCREAMING_SNAKE_CASE_						:					int=0.0,     SCREAMING_SNAKE_CASE_						:					Optional[Any]=True,     SCREAMING_SNAKE_CASE_						:					str="gelu",     SCREAMING_SNAKE_CASE_						:					List[Any]=10_24,     SCREAMING_SNAKE_CASE_						:					int=0.1,     SCREAMING_SNAKE_CASE_						:					List[Any]=0.0,     SCREAMING_SNAKE_CASE_						:					int=0.0,     SCREAMING_SNAKE_CASE_						:					str=0.02,     SCREAMING_SNAKE_CASE_						:					Any=False,     SCREAMING_SNAKE_CASE_						:					List[str]=4,     SCREAMING_SNAKE_CASE_						:					str=20_48,     SCREAMING_SNAKE_CASE_						:					int=20_48,     SCREAMING_SNAKE_CASE_						:					Union[str, Any]=None,     SCREAMING_SNAKE_CASE_						:					Union[str, Any]=False,     **SCREAMING_SNAKE_CASE_						:					Tuple,     ):
										snake_case							:  Optional[int]			=     vocab_size
										snake_case							:  Optional[int]			=     max_position_embeddings
										snake_case							:  Dict			=     hidden_size
										snake_case							:  Any			=     ffn_dim
										snake_case							:  Optional[Any]			=     num_hidden_layers
										snake_case							:  Tuple			=     num_attention_heads
										snake_case							:  int			=     dropout
										snake_case							:  Union[str, Any]			=     attention_dropout
										snake_case							:  Optional[int]			=     activation_dropout
										snake_case							:  Union[str, Any]			=     activation_function
										snake_case							:  Tuple			=     initializer_factor
										snake_case							:  List[Any]			=     layerdrop
										snake_case							:  Optional[int]			=     use_cache
										snake_case							:  Dict			=     scale_embedding  # scale factor will be sqrt(d_model) if True
										snake_case							:  Tuple			=     num_codebooks
										super().__init__(
										    pad_token_id=SCREAMING_SNAKE_CASE_,     bos_token_id=SCREAMING_SNAKE_CASE_,     eos_token_id=SCREAMING_SNAKE_CASE_,     tie_word_embeddings=SCREAMING_SNAKE_CASE_,     **SCREAMING_SNAKE_CASE_,     )




class     a (					__magic_name__     ):
				_snake_case		     =			'''musicgen'''
				_snake_case		     =			True




				def __init__(				self						:					Dict,     **SCREAMING_SNAKE_CASE_						:					Optional[int]   ):
										super().__init__(**SCREAMING_SNAKE_CASE_   )
										if "text_encoder" not in kwargs or "audio_encoder" not in kwargs or "decoder" not in kwargs:
																raise ValueError('''Config has to be initialized with text_encoder, audio_encoder and decoder config'''   )

										snake_case							:  Any			=     kwargs.pop('''text_encoder'''   )
										snake_case							:  List[Any]			=     text_encoder_config.pop('''model_type'''   )

										snake_case							:  Union[str, Any]			=     kwargs.pop('''audio_encoder'''   )
										snake_case							:  Tuple			=     audio_encoder_config.pop('''model_type'''   )

										snake_case							:  List[str]			=     kwargs.pop('''decoder'''   )

										snake_case							:  List[str]			=     AutoConfig.for_model(SCREAMING_SNAKE_CASE_,     **SCREAMING_SNAKE_CASE_   )
										snake_case							:  List[Any]			=     AutoConfig.for_model(SCREAMING_SNAKE_CASE_,     **SCREAMING_SNAKE_CASE_   )
										snake_case							:  Optional[Any]			=     MusicgenDecoderConfig(**SCREAMING_SNAKE_CASE_   )
										snake_case							:  Any			=     True




				@classmethod
				def 						__snake_case				(				cls						:					Tuple,     SCREAMING_SNAKE_CASE_						:					PretrainedConfig,     SCREAMING_SNAKE_CASE_						:					PretrainedConfig,     SCREAMING_SNAKE_CASE_						:					MusicgenDecoderConfig,     **SCREAMING_SNAKE_CASE_						:					Optional[Any],     ):
										return cls(
										    text_encoder=text_encoder_config.to_dict(),     audio_encoder=audio_encoder_config.to_dict(),     decoder=decoder_config.to_dict(),     **SCREAMING_SNAKE_CASE_,     )




				def 						__snake_case				(				self						:					Optional[Any]   ):
										snake_case							:  Dict			=     copy.deepcopy(self.__dict__   )
										snake_case							:  Tuple			=     self.text_encoder.to_dict()
										snake_case							:  str			=     self.audio_encoder.to_dict()
										snake_case							:  Dict			=     self.decoder.to_dict()
										snake_case							:  Optional[int]			=     self.__class__.model_type
										return output

