

from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
  from ...feature_extraction_utils import FeatureExtractionMixin
  from ...tokenization_utils_base import PreTrainedTokenizerBase
  from ...utils import TensorType

lowercase_									=   logging.get_logger(__name__)

lowercase_									=   {
    'openai/whisper-base': 'https://huggingface.co/openai/whisper-base/resolve/main/config.json',
}

# fmt: off
lowercase_									=   [
    1, 2, 7, 8, 9, 1_0, 1_4, 2_5,
    2_6, 2_7, 2_8, 2_9, 3_1, 5_8, 5_9, 6_0, 6_1, 6_2,
    6_3, 9_0, 9_1, 9_2, 9_3, 3_5_7, 3_6_6, 4_3_8, 5_3_2, 6_8_5,
    7_0_5, 7_9_6, 9_3_0, 1_0_5_8, 1_2_2_0, 1_2_6_7, 1_2_7_9, 1_3_0_3, 1_3_4_3, 1_3_7_7,
    1_3_9_1, 1_6_3_5, 1_7_8_2, 1_8_7_5, 2_1_6_2, 2_3_6_1, 2_4_8_8, 3_4_6_7, 4_0_0_8, 4_2_1_1,
    4_6_0_0, 4_8_0_8, 5_2_9_9, 5_8_5_5, 6_3_2_9, 7_2_0_3, 9_6_0_9, 9_9_5_9, 1_0_5_6_3, 1_0_7_8_6,
    1_1_4_2_0, 1_1_7_0_9, 1_1_9_0_7, 1_3_1_6_3, 1_3_6_9_7, 1_3_7_0_0, 1_4_8_0_8, 1_5_3_0_6, 1_6_4_1_0, 1_6_7_9_1,
    1_7_9_9_2, 1_9_2_0_3, 1_9_5_1_0, 2_0_7_2_4, 2_2_3_0_5, 2_2_9_3_5, 2_7_0_0_7, 3_0_1_0_9, 3_0_4_2_0, 3_3_4_0_9,
    3_4_9_4_9, 4_0_2_8_3, 4_0_4_9_3, 4_0_5_4_9, 4_7_2_8_2, 4_9_1_4_6, 5_0_2_5_7, 5_0_3_5_9, 5_0_3_6_0, 5_0_3_6_1
]
lowercase_									=   [
    1, 2, 7, 8, 9, 1_0, 1_4, 2_5,
    2_6, 2_7, 2_8, 2_9, 3_1, 5_8, 5_9, 6_0, 6_1, 6_2,
    6_3, 9_0, 9_1, 9_2, 9_3, 3_5_9, 5_0_3, 5_2_2, 5_4_2, 8_7_3,
    8_9_3, 9_0_2, 9_1_8, 9_2_2, 9_3_1, 1_3_5_0, 1_8_5_3, 1_9_8_2, 2_4_6_0, 2_6_2_7,
    3_2_4_6, 3_2_5_3, 3_2_6_8, 3_5_3_6, 3_8_4_6, 3_9_6_1, 4_1_8_3, 4_6_6_7, 6_5_8_5, 6_6_4_7,
    7_2_7_3, 9_0_6_1, 9_3_8_3, 1_0_4_2_8, 1_0_9_2_9, 1_1_9_3_8, 1_2_0_3_3, 1_2_3_3_1, 1_2_5_6_2, 1_3_7_9_3,
    1_4_1_5_7, 1_4_6_3_5, 1_5_2_6_5, 1_5_6_1_8, 1_6_5_5_3, 1_6_6_0_4, 1_8_3_6_2, 1_8_9_5_6, 2_0_0_7_5, 2_1_6_7_5,
    2_2_5_2_0, 2_6_1_3_0, 2_6_1_6_1, 2_6_4_3_5, 2_8_2_7_9, 2_9_4_6_4, 3_1_6_5_0, 3_2_3_0_2, 3_2_4_7_0, 3_6_8_6_5,
    4_2_8_6_3, 4_7_4_2_5, 4_9_8_7_0, 5_0_2_5_4, 5_0_2_5_8, 5_0_3_6_0, 5_0_3_6_1, 5_0_3_6_2
]





class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):
  _a									=  """whisper"""
  _a									=  ["""past_key_values"""]
  _a									=  {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}




  def __init__(				self							,   lowerCAmelCase=51_865							,   lowerCAmelCase=80							,   lowerCAmelCase=6							,   lowerCAmelCase=4							,   lowerCAmelCase=6							,   lowerCAmelCase=4							,   lowerCAmelCase=1_536							,   lowerCAmelCase=1_536							,   lowerCAmelCase=0.0							,   lowerCAmelCase=0.0							,   lowerCAmelCase=50_257							,   lowerCAmelCase=True							,   lowerCAmelCase=True							,   lowerCAmelCase="gelu"							,   lowerCAmelCase=256							,   lowerCAmelCase=0.0							,   lowerCAmelCase=0.0							,   lowerCAmelCase=0.0							,   lowerCAmelCase=0.02							,   lowerCAmelCase=False							,   lowerCAmelCase=1_500							,   lowerCAmelCase=448							,   lowerCAmelCase=50_256							,   lowerCAmelCase=50_256							,   lowerCAmelCase=50_256							,   lowerCAmelCase=None							,   lowerCAmelCase=[220, 50_256]							,   lowerCAmelCase=False							,   lowerCAmelCase=256							,   lowerCAmelCase=False							,   lowerCAmelCase=0.05							,   lowerCAmelCase=10							,   lowerCAmelCase=2							,   lowerCAmelCase=0.0							,   lowerCAmelCase=10							,   lowerCAmelCase=0							,   lowerCAmelCase=7							,   **lowerCAmelCase							,   )  -> List[Any]:
        '''simple docstring'''






        _lowercase	       =vocab_size
        _lowercase	       =num_mel_bins
        _lowercase	       =d_model
        _lowercase	       =encoder_layers
        _lowercase	       =encoder_attention_heads
        _lowercase	       =decoder_layers
        _lowercase	       =decoder_attention_heads
        _lowercase	       =decoder_ffn_dim
        _lowercase	       =encoder_ffn_dim
        _lowercase	       =dropout
        _lowercase	       =attention_dropout
        _lowercase	       =activation_dropout
        _lowercase	       =activation_function
        _lowercase	       =init_std
        _lowercase	       =encoder_layerdrop
        _lowercase	       =decoder_layerdrop
        _lowercase	       =use_cache
        _lowercase	       =encoder_layers
        _lowercase	       =scale_embedding  # scale factor will be sqrt(d_model) if True
        _lowercase	       =max_source_positions
        _lowercase	       =max_target_positions

        # Audio Classification-specific parameters. Feel free to ignore for other classes.
        _lowercase	       =classifier_proj_size
        _lowercase	       =use_weighted_layer_sum

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        _lowercase	       =apply_spec_augment
        _lowercase	       =mask_time_prob
        _lowercase	       =mask_time_length
        _lowercase	       =mask_time_min_masks
        _lowercase	       =mask_feature_prob
        _lowercase	       =mask_feature_length
        _lowercase	       =mask_feature_min_masks

        _lowercase	       =median_filter_width

        super().__init__(
            pad_token_id=lowerCAmelCase							,   bos_token_id=lowerCAmelCase							,   eos_token_id=lowerCAmelCase							,   is_encoder_decoder=lowerCAmelCase							,   decoder_start_token_id=lowerCAmelCase							,   suppress_tokens=lowerCAmelCase							,   begin_suppress_tokens=lowerCAmelCase							,   **lowerCAmelCase							,   )





class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):




  @property
  def   A__     (				self					)  -> Mapping[str, Mapping[int, str]]:
        '''simple docstring'''






        _lowercase	       =OrderedDict(
            [
                ('input_features', {0: 'batch', 1: 'feature_size', 2: 'encoder_sequence'}),
            ]					)
        if self.use_past:
              _lowercase	       ={0: 'batch'}
        else:
              _lowercase	       ={0: 'batch', 1: 'decoder_sequence'}

        if self.use_past:
              self.fill_with_past_key_values_(lowerCAmelCase							,   direction='inputs'					)

        return common_inputs




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase = -1							,   lowerCAmelCase = -1							,   lowerCAmelCase = False							,   lowerCAmelCase = None							,   lowerCAmelCase = 22_050							,   lowerCAmelCase = 5.0							,   lowerCAmelCase = 220							,   )  -> Mapping[str, Any]:
        '''simple docstring'''






        _lowercase	       =OrderedDict()
        _lowercase	       =OnnxConfig.generate_dummy_inputs(
            self							,   preprocessor=preprocessor.feature_extractor							,   batch_size=lowerCAmelCase							,   framework=lowerCAmelCase							,   sampling_rate=lowerCAmelCase							,   time_duration=lowerCAmelCase							,   frequency=lowerCAmelCase							,   )
        _lowercase	       =encoder_inputs['input_features'].shape[2]
        _lowercase	       =encoder_sequence_length // 2 if self.use_past else seq_length

        _lowercase	       =super().generate_dummy_inputs(
            preprocessor.tokenizer							,   lowerCAmelCase							,   lowerCAmelCase							,   lowerCAmelCase							,   lowerCAmelCase					)

        _lowercase	       =encoder_inputs.pop('input_features'					)
        _lowercase	       =decoder_inputs.pop('decoder_input_ids'					)

        if "past_key_values" in decoder_inputs:
              _lowercase	       =decoder_inputs.pop('past_key_values'					)

        return dummy_inputs




  @property
  def   A__     (				self					)  -> float:
        '''simple docstring'''






        return 1e-3





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


lowercase_									=   logging.getLogger(__name__)





class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase							,   lowerCAmelCase=None							,   lowerCAmelCase=None					)  -> List[str]:
        '''simple docstring'''






        _lowercase	       =self.layer[current_layer](lowerCAmelCase							,   lowerCAmelCase							,   head_mask[current_layer]					)

        _lowercase	       =layer_outputs[0]

        return hidden_states





@add_start_docstrings(
    """The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."""			,			SCREAMING_SNAKE_CASE			,			)
class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):




  def __init__(				self							,   lowerCAmelCase					)  -> Union[str, Any]:
        '''simple docstring'''






        super().__init__(lowerCAmelCase					)

        _lowercase	       =BertEncoderWithPabee(lowerCAmelCase					)

        self.init_weights()
        _lowercase	       =0
        _lowercase	       =0
        _lowercase	       =0

        _lowercase	       =0




  def   A__     (				self							,   lowerCAmelCase					)  -> Optional[Any]:
        '''simple docstring'''






        _lowercase	       =threshold




  def   A__     (				self							,   lowerCAmelCase					)  -> List[Any]:
        '''simple docstring'''






        _lowercase	       =patience




  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =0
        _lowercase	       =0




  def   A__     (				self					)  -> int:
        '''simple docstring'''






        _lowercase	       =self.inference_layers_num / self.inference_instances_num
        _lowercase	       =(
            F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
            F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
        )
        print(lowerCAmelCase					)




  @add_start_docstrings_to_model_forward(lowerCAmelCase					)
  def   A__     (				self							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=False							,   )  -> str:
        '''simple docstring'''






        if input_ids is not None and inputs_embeds is not None:
              raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'					)
        elif input_ids is not None:
              _lowercase	       =input_ids.size()
        elif inputs_embeds is not None:
              _lowercase	       =inputs_embeds.size()[:-1]
        else:
              raise ValueError('You have to specify either input_ids or inputs_embeds'					)

        _lowercase	       =input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
              _lowercase	       =torch.ones(lowerCAmelCase							,   device=lowerCAmelCase					)
        if token_type_ids is None:
              _lowercase	       =torch.zeros(lowerCAmelCase							,   dtype=torch.long							,   device=lowerCAmelCase					)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        _lowercase	       =self.get_extended_attention_mask(lowerCAmelCase							,   lowerCAmelCase							,   lowerCAmelCase					)

        # If a 2D ou 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
              _lowercase       ,       _lowercase       ,       _lowercase	       =encoder_hidden_states.size()
              _lowercase	       =(encoder_batch_size, encoder_sequence_length)
              if encoder_attention_mask is None:
                    _lowercase	       =torch.ones(lowerCAmelCase							,   device=lowerCAmelCase					)
              _lowercase	       =self.invert_attention_mask(lowerCAmelCase					)
        else:
              _lowercase	       =None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        _lowercase	       =self.get_head_mask(lowerCAmelCase							,   self.config.num_hidden_layers					)

        _lowercase	       =self.embeddings(
            input_ids=lowerCAmelCase							,   position_ids=lowerCAmelCase							,   token_type_ids=lowerCAmelCase							,   inputs_embeds=lowerCAmelCase					)
        _lowercase	       =embedding_output

        if self.training:
              _lowercase	       =[]
              for i in range(self.config.num_hidden_layers					):
                    _lowercase	       =self.encoder.adaptive_forward(
                        lowerCAmelCase							,   current_layer=lowerCAmelCase							,   attention_mask=lowerCAmelCase							,   head_mask=lowerCAmelCase					)

                    _lowercase	       =self.pooler(lowerCAmelCase					)
                    _lowercase	       =output_layers[i](output_dropout(lowerCAmelCase					)					)
                    res.append(lowerCAmelCase					)
        elif self.patience == 0:  # Use all layers for inference
              _lowercase	       =self.encoder(
                  lowerCAmelCase							,   attention_mask=lowerCAmelCase							,   head_mask=lowerCAmelCase							,   encoder_hidden_states=lowerCAmelCase							,   encoder_attention_mask=lowerCAmelCase							,   )
              _lowercase	       =self.pooler(encoder_outputs[0]					)
              _lowercase	       =[output_layers[self.config.num_hidden_layers - 1](lowerCAmelCase					)]
        else:
              _lowercase	       =0
              _lowercase	       =None
              _lowercase	       =0
              for i in range(self.config.num_hidden_layers					):
                    calculated_layer_num += 1
                    _lowercase	       =self.encoder.adaptive_forward(
                        lowerCAmelCase							,   current_layer=lowerCAmelCase							,   attention_mask=lowerCAmelCase							,   head_mask=lowerCAmelCase					)

                    _lowercase	       =self.pooler(lowerCAmelCase					)
                    _lowercase	       =output_layers[i](lowerCAmelCase					)
                    if regression:
                          _lowercase	       =logits.detach()
                          if patient_result is not None:
                                _lowercase	       =patient_result.detach()
                          if (patient_result is not None) and torch.abs(patient_result - labels					) < self.regression_threshold:
                                patient_counter += 1
                          else:
                                _lowercase	       =0
                    else:
                          _lowercase	       =logits.detach().argmax(dim=1					)
                          if patient_result is not None:
                                _lowercase	       =patient_result.detach().argmax(dim=1					)
                          if (patient_result is not None) and torch.all(labels.eq(lowerCAmelCase					)					):
                                patient_counter += 1
                          else:
                                _lowercase	       =0

                    _lowercase	       =logits
                    if patient_counter == self.patience:
                          break
              _lowercase	       =[patient_result]
              self.inference_layers_num += calculated_layer_num
              self.inference_instances_num += 1

        return res





@add_start_docstrings(
    """Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """			,			SCREAMING_SNAKE_CASE			,			)
class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):




  def __init__(				self							,   lowerCAmelCase					)  -> List[Any]:
        '''simple docstring'''






        super().__init__(lowerCAmelCase					)
        _lowercase	       =config.num_labels

        _lowercase	       =BertModelWithPabee(lowerCAmelCase					)
        _lowercase	       =nn.Dropout(config.hidden_dropout_prob					)
        _lowercase	       =nn.ModuleList(
            [nn.Linear(config.hidden_size							,   self.config.num_labels					) for _ in range(config.num_hidden_layers					)]					)

        self.init_weights()




  @add_start_docstrings_to_model_forward(lowerCAmelCase					)
  def   A__     (				self							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   )  -> Union[str, Any]:
        '''simple docstring'''






        _lowercase	       =self.bert(
            input_ids=lowerCAmelCase							,   attention_mask=lowerCAmelCase							,   token_type_ids=lowerCAmelCase							,   position_ids=lowerCAmelCase							,   head_mask=lowerCAmelCase							,   inputs_embeds=lowerCAmelCase							,   output_dropout=self.dropout							,   output_layers=self.classifiers							,   regression=self.num_labels == 1							,   )

        _lowercase	       =(logits[-1],)

        if labels is not None:
              _lowercase	       =None
              _lowercase	       =0
              for ix, logits_item in enumerate(lowerCAmelCase					):
                    if self.num_labels == 1:
                          #  We are doing regression
                          _lowercase	       =MSELoss()
                          _lowercase	       =loss_fct(logits_item.view(-1					)							,   labels.view(-1					)					)
                    else:
                          _lowercase	       =CrossEntropyLoss()
                          _lowercase	       =loss_fct(logits_item.view(-1							,   self.num_labels					)							,   labels.view(-1					)					)
                    if total_loss is None:
                          _lowercase	       =loss
                    else:
                          total_loss += loss * (ix + 1)
                    total_weights += ix + 1
              _lowercase	       =(total_loss / total_weights,) + outputs

        return outputs


