



'''simple docstring'''


from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
   from ...feature_extraction_utils import FeatureExtractionMixin
   from ...tokenization_utils_base import PreTrainedTokenizerBase
   from ...utils import TensorType

_lowerCamelCase	:       Union[str, Any]      	=	logging.get_logger(__name__)

_lowerCamelCase	:       Tuple      	=	{
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}

# fmt: off
_lowerCamelCase	:       Dict      	=	[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 1_0563, 1_0786,
    1_1420, 1_1709, 1_1907, 1_3163, 1_3697, 1_3700, 1_4808, 1_5306, 1_6410, 1_6791,
    1_7992, 1_9203, 1_9510, 2_0724, 2_2305, 2_2935, 2_7007, 3_0109, 3_0420, 3_3409,
    3_4949, 4_0283, 4_0493, 4_0549, 4_7282, 4_9146, 5_0257, 5_0359, 5_0360, 5_0361
]
_lowerCamelCase	:       List[str]      	=	[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 1_0428, 1_0929, 1_1938, 1_2033, 1_2331, 1_2562, 1_3793,
    1_4157, 1_4635, 1_5265, 1_5618, 1_6553, 1_6604, 1_8362, 1_8956, 2_0075, 2_1675,
    2_2520, 2_6130, 2_6161, 2_6435, 2_8279, 2_9464, 3_1650, 3_2302, 3_2470, 3_6865,
    4_2863, 4_7425, 4_9870, 5_0254, 5_0258, 5_0360, 5_0361, 5_0362
]

class  SCREAMING_SNAKE_CASE (  _a		):






    """simple docstring"""





    _SCREAMING_SNAKE_CASE	  =  """whisper"""
    _SCREAMING_SNAKE_CASE	  =  ["""past_key_values"""]
    _SCREAMING_SNAKE_CASE	  =  {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}






    def __init__(  self      :					Optional[Any]  ,						UpperCamelCase__      :					List[str]=5_1_8_6_5  ,						UpperCamelCase__      :					str=8_0  ,						UpperCamelCase__      :					Tuple=6  ,						UpperCamelCase__      :					Optional[int]=4  ,						UpperCamelCase__      :					int=6  ,						UpperCamelCase__      :					Optional[int]=4  ,						UpperCamelCase__      :					Optional[Any]=1_5_3_6  ,						UpperCamelCase__      :					int=1_5_3_6  ,						UpperCamelCase__      :					Union[str, Any]=0.0  ,						UpperCamelCase__      :					Tuple=0.0  ,						UpperCamelCase__      :					Tuple=5_0_2_5_7  ,						UpperCamelCase__      :					Union[str, Any]=True  ,						UpperCamelCase__      :					List[str]=True  ,						UpperCamelCase__      :					Optional[int]="gelu"  ,						UpperCamelCase__      :					Tuple=2_5_6  ,						UpperCamelCase__      :					List[str]=0.0  ,						UpperCamelCase__      :					List[Any]=0.0  ,						UpperCamelCase__      :					Any=0.0  ,						UpperCamelCase__      :					Optional[int]=0.0_2  ,						UpperCamelCase__      :					Any=False  ,						UpperCamelCase__      :					Tuple=1_5_0_0  ,						UpperCamelCase__      :					List[str]=4_4_8  ,						UpperCamelCase__      :					List[Any]=5_0_2_5_6  ,						UpperCamelCase__      :					Any=5_0_2_5_6  ,						UpperCamelCase__      :					Dict=5_0_2_5_6  ,						UpperCamelCase__      :					Optional[Any]=None  ,						UpperCamelCase__      :					Tuple=[2_2_0, 5_0_2_5_6]  ,						UpperCamelCase__      :					int=False  ,						UpperCamelCase__      :					List[str]=2_5_6  ,						UpperCamelCase__      :					str=False  ,						UpperCamelCase__      :					Tuple=0.0_5  ,						UpperCamelCase__      :					Any=1_0  ,						UpperCamelCase__      :					Dict=2  ,						UpperCamelCase__      :					Union[str, Any]=0.0  ,						UpperCamelCase__      :					str=1_0  ,						UpperCamelCase__      :					List[Any]=0  ,						UpperCamelCase__      :					Tuple=7  ,						**UpperCamelCase__      :					Tuple  ,						):


       """simple docstring"""







       UpperCamelCase					=  vocab_size
       UpperCamelCase					=  num_mel_bins
       UpperCamelCase					=  d_model
       UpperCamelCase					=  encoder_layers
       UpperCamelCase					=  encoder_attention_heads
       UpperCamelCase					=  decoder_layers
       UpperCamelCase					=  decoder_attention_heads
       UpperCamelCase					=  decoder_ffn_dim
       UpperCamelCase					=  encoder_ffn_dim
       UpperCamelCase					=  dropout
       UpperCamelCase					=  attention_dropout
       UpperCamelCase					=  activation_dropout
       UpperCamelCase					=  activation_function
       UpperCamelCase					=  init_std
       UpperCamelCase					=  encoder_layerdrop
       UpperCamelCase					=  decoder_layerdrop
       UpperCamelCase					=  use_cache
       UpperCamelCase					=  encoder_layers
       UpperCamelCase					=  scale_embedding  # scale factor will be sqrt(d_model) if True
       UpperCamelCase					=  max_source_positions
       UpperCamelCase					=  max_target_positions

       # Audio Classification-specific parameters. Feel free to ignore for other classes.
       UpperCamelCase					=  classifier_proj_size
       UpperCamelCase					=  use_weighted_layer_sum

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       UpperCamelCase					=  apply_spec_augment
       UpperCamelCase					=  mask_time_prob
       UpperCamelCase					=  mask_time_length
       UpperCamelCase					=  mask_time_min_masks
       UpperCamelCase					=  mask_feature_prob
       UpperCamelCase					=  mask_feature_length
       UpperCamelCase					=  mask_feature_min_masks

       UpperCamelCase					=  median_filter_width

       super().__init__(
           pad_token_id=UpperCamelCase__  ,						bos_token_id=UpperCamelCase__  ,						eos_token_id=UpperCamelCase__  ,						is_encoder_decoder=UpperCamelCase__  ,						decoder_start_token_id=UpperCamelCase__  ,						suppress_tokens=UpperCamelCase__  ,						begin_suppress_tokens=UpperCamelCase__  ,						**UpperCamelCase__  ,						)

class  SCREAMING_SNAKE_CASE (  _a		):






    """simple docstring"""





    @property
    def  A       (  self      :					Union[str, Any] ):


       """simple docstring"""







       UpperCamelCase					=  OrderedDict(
           [
               ('input_features', {0: 'batch', 1: 'feature_size', 2: 'encoder_sequence'}),
           ] )
       if self.use_past:
          UpperCamelCase					=  {0: 'batch'}
       else:
          UpperCamelCase					=  {0: 'batch', 1: 'decoder_sequence'}

       if self.use_past:
          self.fill_with_past_key_values_(UpperCamelCase__  ,						direction='inputs' )

       return common_inputs






    def  A       (  self      :					Optional[int]  ,						UpperCamelCase__      :					Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					bool = False  ,						UpperCamelCase__      :					Optional["TensorType"] = None  ,						UpperCamelCase__      :					int = 2_2_0_5_0  ,						UpperCamelCase__      :					float = 5.0  ,						UpperCamelCase__      :					int = 2_2_0  ,						):


       """simple docstring"""







       UpperCamelCase					=  OrderedDict()
       UpperCamelCase					=  OnnxConfig.generate_dummy_inputs(
           self  ,						preprocessor=preprocessor.feature_extractor  ,						batch_size=UpperCamelCase__  ,						framework=UpperCamelCase__  ,						sampling_rate=UpperCamelCase__  ,						time_duration=UpperCamelCase__  ,						frequency=UpperCamelCase__  ,						)
       UpperCamelCase					=  encoder_inputs['input_features'].shape[2]
       UpperCamelCase					=  encoder_sequence_length // 2 if self.use_past else seq_length

       UpperCamelCase					=  super().generate_dummy_inputs(
           preprocessor.tokenizer  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )

       UpperCamelCase					=  encoder_inputs.pop('input_features' )
       UpperCamelCase					=  decoder_inputs.pop('decoder_input_ids' )

       if "past_key_values" in decoder_inputs:
          UpperCamelCase					=  decoder_inputs.pop('past_key_values' )

       return dummy_inputs






    @property
    def  A       (  self      :					Dict ):


       """simple docstring"""







       return 1E-3



from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
				from fairscale.optim import OSS


__snake_case								=  logging.get_logger(__name__)

__snake_case								=  {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}




class 						lowercase__  (						_UpperCAmelCase  ):
	def __init__(			self							:   str       ,   UpperCAmelCase_							:   int=None       ,   UpperCAmelCase_							:   List[str]=None       ,   *UpperCAmelCase_							:   Union[str, Any]       ,   **UpperCAmelCase_							:   Optional[Any]							):
						super().__init__(*UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if config is None:
											assert isinstance(self.model       ,   UpperCAmelCase_							), (
											    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
											    F' {self.model.__class__}'
											)
											SCREAMING_SNAKE_CASE__													= self.model.config
						else:
											SCREAMING_SNAKE_CASE__													= config

						SCREAMING_SNAKE_CASE__													= data_args
						SCREAMING_SNAKE_CASE__													= self.config.tgt_vocab_size if isinstance(self.config       ,   UpperCAmelCase_							) else self.config.vocab_size

						if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
											assert self.config.pad_token_id is not None, (
											    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
											    " calculation or doing label smoothing."
											)

						if self.config.pad_token_id is None and self.config.eos_token_id is not None:
											logger.warning(
											    F'The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'
											    ' padding..'							)

						if self.args.label_smoothing == 0:
											SCREAMING_SNAKE_CASE__													= torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id							)
						else:
											# dynamically import label_smoothed_nll_loss
											from utils import label_smoothed_nll_loss

											SCREAMING_SNAKE_CASE__													= label_smoothed_nll_loss
	def      A_				(			self							:   Tuple       ,   UpperCAmelCase_							:   int							):
						if self.optimizer is None:
											SCREAMING_SNAKE_CASE__													= ['bias', 'LayerNorm.weight']
											SCREAMING_SNAKE_CASE__													= [
											    {
											        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay							)],
											        'weight_decay': self.args.weight_decay,
											    },
											    {
											        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay							)],
											        'weight_decay': 0.0,
											    },
											]
											SCREAMING_SNAKE_CASE__													= Adafactor if self.args.adafactor else AdamW
											if self.args.adafactor:
																SCREAMING_SNAKE_CASE__													= Adafactor
																SCREAMING_SNAKE_CASE__													= {'scale_parameter': False, 'relative_step': False}
											else:
																SCREAMING_SNAKE_CASE__													= AdamW
																SCREAMING_SNAKE_CASE__													= {
																    'betas': (self.args.adam_betaa, self.args.adam_betaa),
																    'eps': self.args.adam_epsilon,
																}
											SCREAMING_SNAKE_CASE__													= self.args.learning_rate
											if self.sharded_ddp:
																SCREAMING_SNAKE_CASE__													= OSS(
																    params=UpperCAmelCase_       ,   optim=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )
											else:
																SCREAMING_SNAKE_CASE__													= optimizer_cls(UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if self.lr_scheduler is None:
											SCREAMING_SNAKE_CASE__													= self._get_lr_scheduler(UpperCAmelCase_							)
						else:  # ignoring --lr_scheduler
											logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'							)
	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= arg_to_scheduler[self.args.lr_scheduler]
						if self.args.lr_scheduler == "constant":
											SCREAMING_SNAKE_CASE__													= schedule_func(self.optimizer							)
						elif self.args.lr_scheduler == "constant_w_warmup":
											SCREAMING_SNAKE_CASE__													= schedule_func(self.optimizer       ,   num_warmup_steps=self.args.warmup_steps							)
						else:
											SCREAMING_SNAKE_CASE__													= schedule_func(
											    self.optimizer       ,   num_warmup_steps=self.args.warmup_steps       ,   num_training_steps=UpperCAmelCase_							)
						return scheduler
	def      A_				(			self							:   List[str]							):
						if isinstance(self.train_dataset       ,   torch.utils.data.IterableDataset							):
											return None
						elif is_torch_tpu_available():
											return get_tpu_sampler(self.train_dataset							)
						else:
											if self.args.sortish_sampler:
																self.train_dataset.make_sortish_sampler(
																    self.args.per_device_train_batch_size       ,   distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)       ,   )

											return (
											    RandomSampler(self.train_dataset							)
											    if self.args.local_rank == -1
											    else DistributedSampler(self.train_dataset							)
											)
	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   Tuple       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Dict							):
						if self.args.label_smoothing == 0:
											if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																# force training to ignore pad token
																SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   use_cache=UpperCAmelCase_							)[0]
																SCREAMING_SNAKE_CASE__													= self.loss_fn(logits.view(-1       ,   logits.shape[-1]							)       ,   labels.view(-1							)							)
											else:
																# compute usual loss via models
																SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   labels=UpperCAmelCase_       ,   use_cache=UpperCAmelCase_							)[:2]
						else:
											# compute label smoothed loss
											SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   use_cache=UpperCAmelCase_							)[0]
											SCREAMING_SNAKE_CASE__													= torch.nn.functional.log_softmax(UpperCAmelCase_       ,   dim=-1							)
											SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.loss_fn(UpperCAmelCase_       ,   UpperCAmelCase_       ,   self.args.label_smoothing       ,   ignore_index=self.config.pad_token_id							)
						return loss, logits
	def      A_				(			self							:   Union[str, Any]       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= inputs.pop('labels'							)
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self._compute_loss(UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
						return loss
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   nn.Module       ,   UpperCAmelCase_							:   Dict[str, Union[torch.Tensor, Any]]       ,   UpperCAmelCase_							:   bool       ,   UpperCAmelCase_							:   Optional[List[str]] = None       ,   ):
						SCREAMING_SNAKE_CASE__													= self._prepare_inputs(UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= {
						    'max_length': self.data_args.val_max_target_length
						    if self.data_args is not None
						    else self.config.max_length,
						    'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
						}

						if self.args.predict_with_generate and not self.args.prediction_loss_only:
											SCREAMING_SNAKE_CASE__													= self.model.generate(
											    inputs['input_ids']       ,   attention_mask=inputs['attention_mask']       ,   **UpperCAmelCase_       ,   )
											# in case the batch is shorter than max length, the output should be padded
											if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																SCREAMING_SNAKE_CASE__													= self._pad_tensors_to_max_len(UpperCAmelCase_       ,   gen_kwargs['max_length']							)

						SCREAMING_SNAKE_CASE__													= inputs.pop('labels'							)
						with torch.no_grad():
											# compute loss on predict data
											SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self._compute_loss(UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= loss.mean().detach()
						if self.args.prediction_loss_only:
											return (loss, None, None)

						SCREAMING_SNAKE_CASE__													= generated_tokens if self.args.predict_with_generate else logits

						if labels.shape[-1] < gen_kwargs["max_length"]:
											SCREAMING_SNAKE_CASE__													= self._pad_tensors_to_max_len(UpperCAmelCase_       ,   gen_kwargs['max_length']							)

						return (loss, logits, labels)





	def      A_				(			self							:   Union[str, Any]       ,   UpperCAmelCase_							:   Tuple       ,   UpperCAmelCase_							:   List[str]							):
						# If PAD token is not defined at least EOS token has to be defined
						SCREAMING_SNAKE_CASE__													= self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

						if pad_token_id is None:
											raise ValueError(
											    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
											    F' padded to `max_length`={max_length}'							)

						SCREAMING_SNAKE_CASE__													= pad_token_id * torch.ones(
						    (tensor.shape[0], max_length)       ,   dtype=tensor.dtype       ,   device=tensor.device							)
						SCREAMING_SNAKE_CASE__													= tensor
						return padded_tensor





