


from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class        snake_case_   (lowerCamelCase_	,						lowerCamelCase_	,						lowerCamelCase_      ):
					UpperCAmelCase__					:    int          =			[R'''h\.\d+\.attn\.bias''', R'''h\.\d+\.attn\.masked_bias''']
					@register_to_config
					def __init__(      self   :Tuple			,__snake_case   :int			,__snake_case   :int			,__snake_case   :Optional[int] = None			,__snake_case   :int = 5_02_57			,__snake_case   :int = 10_24			,__snake_case   :int = 7_68			,__snake_case   :int = 12			,__snake_case   :int = 12			,__snake_case   :Optional[int] = None			,__snake_case   :str = "gelu_new"			,__snake_case   :float = 0.1			,__snake_case   :float = 0.1			,__snake_case   :float = 0.1			,__snake_case   :float = 1E-5			,__snake_case   :float = 0.02			,__snake_case   :bool = True			,__snake_case   :bool = True			,__snake_case   :bool = False			,__snake_case   :bool = False			,)					->    List[Any]:
												super().__init__()

												a__             =     prefix_length

												if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
																			raise ValueError(
																			    F'`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'
																			    F' `n_embd`: {n_embd} are not equal.'      )

												a__             =     prefix_inner_dim
												a__             =     prefix_hidden_dim

												a__             =     (
												    nn.Linear(self.prefix_inner_dim			,self.prefix_hidden_dim      )
												    if self.prefix_hidden_dim is not None
												    else nn.Identity()
												)
												a__             =     (
												    nn.Linear(self.prefix_hidden_dim			,__snake_case      ) if self.prefix_hidden_dim is not None else nn.Identity()
												)

												a__             =     GPTaConfig(
												    vocab_size=__snake_case			,n_positions=__snake_case			,n_embd=__snake_case			,n_layer=__snake_case			,n_head=__snake_case			,n_inner=__snake_case			,activation_function=__snake_case			,resid_pdrop=__snake_case			,embd_pdrop=__snake_case			,attn_pdrop=__snake_case			,layer_norm_epsilon=__snake_case			,initializer_range=__snake_case			,scale_attn_weights=__snake_case			,use_cache=__snake_case			,scale_attn_by_inverse_layer_idx=__snake_case			,reorder_and_upcast_attn=__snake_case			,)
												a__             =     GPTaLMHeadModel(__snake_case      )
					def  lowerCamelCase__(      self   :Optional[Any]			,__snake_case   :torch.Tensor			,__snake_case   :torch.Tensor			,__snake_case   :Optional[torch.Tensor] = None			,__snake_case   :Optional[torch.Tensor] = None			,)					->    Any:
												a__             =     self.transformer.transformer.wte(__snake_case      )
												a__             =     self.encode_prefix(__snake_case      )
												a__             =     self.decode_prefix(__snake_case      )
												a__             =     torch.cat((prefix_embeds, embedding_text)			,dim=1      )

												if labels is not None:
																			a__             =     self.get_dummy_token(input_ids.shape[0]			,input_ids.device      )
																			a__             =     torch.cat((dummy_token, input_ids)			,dim=1      )
												a__             =     self.transformer(inputs_embeds=__snake_case			,labels=__snake_case			,attention_mask=__snake_case      )
												if self.prefix_hidden_dim is not None:
																			return out, hidden
												else:
																			return out
					def  lowerCamelCase__(      self   :str			,__snake_case   :int			,__snake_case   :torch.device      )					->    torch.Tensor:
												return torch.zeros(__snake_case			,self.prefix_length			,dtype=torch.intaa			,device=__snake_case      )
					def  lowerCamelCase__(      self   :List[str]			,__snake_case   :List[Any]      )					->    Union[str, Any]:
												return self.encode_prefix(__snake_case      )
					@torch.no_grad()
					def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :int			,__snake_case   :Any			,__snake_case   :List[str]      )					->    str:
												a__             =     torch.split(__snake_case			,1			,dim=0      )
												a__             =     []
												a__             =     []
												for feature in features:
																			a__             =     self.decode_prefix(feature.to(__snake_case      )      )  # back to the clip feature
																			# Only support beam search for now
																			a__      ,					a__             =     self.generate_beam(
																			    input_embeds=__snake_case			,device=__snake_case			,eos_token_id=__snake_case      )
																			generated_tokens.append(output_tokens[0]      )
																			generated_seq_lengths.append(seq_lengths[0]      )
												a__             =     torch.stack(__snake_case      )
												a__             =     torch.stack(__snake_case      )
												return generated_tokens, generated_seq_lengths



					@torch.no_grad()
					def  lowerCamelCase__(      self   :int			,__snake_case   :int=None			,__snake_case   :int=None			,__snake_case   :Union[str, Any]=None			,__snake_case   :int = 5			,__snake_case   :int = 67			,__snake_case   :float = 1.0			,__snake_case   :Optional[int] = None			,)					->    List[str]:
												a__             =     eos_token_id
												a__             =     None
												a__             =     None
												a__             =     torch.ones(__snake_case			,device=__snake_case			,dtype=torch.int      )
												a__             =     torch.zeros(__snake_case			,device=__snake_case			,dtype=torch.bool      )

												if input_embeds is not None:
																			a__             =     input_embeds
												else:
																			a__             =     self.transformer.transformer.wte(__snake_case      )

												for i in range(__snake_case      ):
																			a__             =     self.transformer(inputs_embeds=__snake_case      )
																			a__             =     outputs.logits
																			a__             =     logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
																			a__             =     logits.softmax(-1      ).log()

																			if scores is None:
																										a__      ,					a__             =     logits.topk(__snake_case			,-1      )
																										a__             =     generated.expand(__snake_case			,*generated.shape[1:]      )
																										a__      ,					a__             =     next_tokens.permute(1			,0      ), scores.squeeze(0      )
																										if tokens is None:
																																	a__             =     next_tokens
																										else:
																																	a__             =     tokens.expand(__snake_case			,*tokens.shape[1:]      )
																																	a__             =     torch.cat((tokens, next_tokens)			,dim=1      )
																			else:
																										a__             =     -float(np.inf      )
																										a__             =     0
																										a__             =     scores[:, None] + logits
																										seq_lengths[~is_stopped] += 1
																										a__             =     scores_sum / seq_lengths[:, None]
																										a__      ,					a__             =     scores_sum_average.view(-1      ).topk(__snake_case			,-1      )
																										a__             =     next_tokens // scores_sum.shape[1]
																										a__             =     seq_lengths[next_tokens_source]
																										a__             =     next_tokens % scores_sum.shape[1]
																										a__             =     next_tokens.unsqueeze(1      )
																										a__             =     tokens[next_tokens_source]
																										a__             =     torch.cat((tokens, next_tokens)			,dim=1      )
																										a__             =     generated[next_tokens_source]
																										a__             =     scores_sum_average * seq_lengths
																										a__             =     is_stopped[next_tokens_source]

																			a__             =     self.transformer.transformer.wte(next_tokens.squeeze()      ).view(generated.shape[0]			,1			,-1      )
																			a__             =     torch.cat((generated, next_token_embed)			,dim=1      )
																			a__             =     is_stopped + next_tokens.eq(__snake_case      ).squeeze()
																			if is_stopped.all():
																										break

												a__             =     scores / seq_lengths
												a__             =     scores.argsort(descending=__snake_case      )
												# tokens tensors are already padded to max_seq_length
												a__             =     [tokens[i] for i in order]
												a__             =     torch.stack(__snake_case			,dim=0      )
												a__             =     torch.tensor([seq_lengths[i] for i in order]			,dtype=seq_lengths.dtype      )
												return output_texts, seq_lengths








import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, ClassLabel, Features
from .base import TaskTemplate







@dataclass(frozen=lowerCamelCase_      )
class        snake_case_   (lowerCamelCase_      ):
					UpperCAmelCase__					:    str          =			field(default='''audio-classification'''	,						metadata={'''include_in_asdict_even_if_is_default''': True}      )
					UpperCAmelCase__					:    ClassVar[Features]          =			Features({'''audio''': Audio()}      )
					UpperCAmelCase__					:    ClassVar[Features]          =			Features({'''labels''': ClassLabel}      )
					UpperCAmelCase__					:    str          =			"audio"
					UpperCAmelCase__					:    str          =			"labels"
					def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :int      )					->    str:
												if self.label_column not in features:
																			raise ValueError(F'Column {self.label_column} is not present in features.'      )
												if not isinstance(features[self.label_column]			,__snake_case      ):
																			raise ValueError(F'Column {self.label_column} is not a ClassLabel.'      )
												a__             =     copy.deepcopy(self      )
												a__             =     self.label_schema.copy()
												a__             =     features[self.label_column]
												a__             =     label_schema
												return task_template



					@property
					def  lowerCamelCase__(      self   :Dict      )					->    Dict[str, str]:
												return {
												    self.audio_column: "audio",
												    self.label_column: "labels",
												}




