




'''simple docstring'''




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


a_   :	int							   = logging.getLogger(__name__)


class 						__UpperCamelCase (				lowerCamelCase__       ):






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase,	lowerCAmelCase=None,	lowerCAmelCase=None						):



              """simple docstring"""







              if current_layer == 0:
                     lowerCamelCase_							=self.embedding_hidden_mapping_in(lowerCAmelCase						)
              else:
                     lowerCamelCase_							=hidden_states[0]

              lowerCamelCase_							=int(self.config.num_hidden_layers / self.config.num_hidden_groups						)

              # Index of the hidden group
              lowerCamelCase_							=int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)						)

              lowerCamelCase_							=self.albert_layer_groups[group_idx](
                  lowerCAmelCase,	lowerCAmelCase,	head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group],	)
              lowerCamelCase_							=layer_group_output[0]

              return (hidden_states,)





@add_start_docstrings(
    'The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'    ,      lowerCamelCase__    ,      )
class 						__UpperCamelCase (				lowerCamelCase__       ):






       def __init__(							self,	lowerCAmelCase						):



              """simple docstring"""







              super().__init__(lowerCAmelCase						)

              lowerCamelCase_							=AlbertTransformerWithPabee(lowerCAmelCase						)

              self.init_weights()
              lowerCamelCase_							=0
              lowerCamelCase_							=0
              lowerCamelCase_							=0

              lowerCamelCase_							=0






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=threshold






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=patience






       def   lowercase__  (							self						):



              """simple docstring"""







              lowerCamelCase_							=0
              lowerCamelCase_							=0






       def   lowercase__  (							self						):



              """simple docstring"""







              lowerCamelCase_							=self.inference_layers_num / self.inference_instances_num
              lowerCamelCase_							=(
                  f'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
                  f''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
              )
              print(lowerCAmelCase						)






       @add_start_docstrings_to_model_forward(lowerCAmelCase						)
       def   lowercase__  (							self,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=False,	):



              """simple docstring"""







              if input_ids is not None and inputs_embeds is not None:
                     raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''						)
              elif input_ids is not None:
                     lowerCamelCase_							=input_ids.size()
              elif inputs_embeds is not None:
                     lowerCamelCase_							=inputs_embeds.size()[:-1]
              else:
                     raise ValueError('''You have to specify either input_ids or inputs_embeds'''						)

              lowerCamelCase_							=input_ids.device if input_ids is not None else inputs_embeds.device

              if attention_mask is None:
                     lowerCamelCase_							=torch.ones(lowerCAmelCase,	device=lowerCAmelCase						)
              if token_type_ids is None:
                     lowerCamelCase_							=torch.zeros(lowerCAmelCase,	dtype=torch.long,	device=lowerCAmelCase						)

              lowerCamelCase_							=attention_mask.unsqueeze(1						).unsqueeze(2						)
              lowerCamelCase_							=extended_attention_mask.to(dtype=self.dtype						)  # fp16 compatibility
              lowerCamelCase_							=(1.0 - extended_attention_mask) * -1_0_0_0_0.0
              lowerCamelCase_							=self.get_head_mask(lowerCAmelCase,	self.config.num_hidden_layers						)

              lowerCamelCase_							=self.embeddings(
                  lowerCAmelCase,	position_ids=lowerCAmelCase,	token_type_ids=lowerCAmelCase,	inputs_embeds=lowerCAmelCase						)
              lowerCamelCase_							=embedding_output

              if self.training:
                     lowerCamelCase_							=[]
                     for i in range(self.config.num_hidden_layers						):
                            lowerCamelCase_							=self.encoder.adaptive_forward(
                                lowerCAmelCase,	current_layer=lowerCAmelCase,	attention_mask=lowerCAmelCase,	head_mask=lowerCAmelCase,	)

                            lowerCamelCase_							=self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
                            lowerCamelCase_							=output_layers[i](output_dropout(lowerCAmelCase						)						)
                            res.append(lowerCAmelCase						)
              elif self.patience == 0:  # Use all layers for inference
                     lowerCamelCase_							=self.encoder(lowerCAmelCase,	lowerCAmelCase,	head_mask=lowerCAmelCase						)
                     lowerCamelCase_							=self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
                     lowerCamelCase_							=[output_layers[self.config.num_hidden_layers - 1](lowerCAmelCase						)]
              else:
                     lowerCamelCase_							=0
                     lowerCamelCase_							=None
                     lowerCamelCase_							=0
                     for i in range(self.config.num_hidden_layers						):
                            calculated_layer_num += 1
                            lowerCamelCase_							=self.encoder.adaptive_forward(
                                lowerCAmelCase,	current_layer=lowerCAmelCase,	attention_mask=lowerCAmelCase,	head_mask=lowerCAmelCase,	)

                            lowerCamelCase_							=self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
                            lowerCamelCase_							=output_layers[i](lowerCAmelCase						)
                            if regression:
                                   lowerCamelCase_							=logits.detach()
                                   if patient_result is not None:
                                          lowerCamelCase_							=patient_result.detach()
                                   if (patient_result is not None) and torch.abs(patient_result - labels						) < self.regression_threshold:
                                          patient_counter += 1
                                   else:
                                          lowerCamelCase_							=0
                            else:
                                   lowerCamelCase_							=logits.detach().argmax(dim=1						)
                                   if patient_result is not None:
                                          lowerCamelCase_							=patient_result.detach().argmax(dim=1						)
                                   if (patient_result is not None) and torch.all(labels.eq(lowerCAmelCase						)						):
                                          patient_counter += 1
                                   else:
                                          lowerCamelCase_							=0

                            lowerCamelCase_							=logits
                            if patient_counter == self.patience:
                                   break
                     lowerCamelCase_							=[patient_result]
                     self.inference_layers_num += calculated_layer_num
                     self.inference_instances_num += 1

              return res





@add_start_docstrings(
    'Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '    ,      lowerCamelCase__    ,      )
class 						__UpperCamelCase (				lowerCamelCase__       ):






       def __init__(							self,	lowerCAmelCase						):



              """simple docstring"""







              super().__init__(lowerCAmelCase						)
              lowerCamelCase_							=config.num_labels

              lowerCamelCase_							=AlbertModelWithPabee(lowerCAmelCase						)
              lowerCamelCase_							=nn.Dropout(config.classifier_dropout_prob						)
              lowerCamelCase_							=nn.ModuleList(
                  [nn.Linear(config.hidden_size,	self.config.num_labels						) for _ in range(config.num_hidden_layers						)]						)

              self.init_weights()






       @add_start_docstrings_to_model_forward(lowerCAmelCase						)
       def   lowercase__  (							self,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	lowerCAmelCase=None,	):



              """simple docstring"""







              lowerCamelCase_							=self.albert(
                  input_ids=lowerCAmelCase,	attention_mask=lowerCAmelCase,	token_type_ids=lowerCAmelCase,	position_ids=lowerCAmelCase,	head_mask=lowerCAmelCase,	inputs_embeds=lowerCAmelCase,	output_dropout=self.dropout,	output_layers=self.classifiers,	regression=self.num_labels == 1,	)

              lowerCamelCase_							=(logits[-1],)

              if labels is not None:
                     lowerCamelCase_							=None
                     lowerCamelCase_							=0
                     for ix, logits_item in enumerate(lowerCAmelCase						):
                            if self.num_labels == 1:
                                   #  We are doing regression
                                   lowerCamelCase_							=MSELoss()
                                   lowerCamelCase_							=loss_fct(logits_item.view(-1						),	labels.view(-1						)						)
                            else:
                                   lowerCamelCase_							=CrossEntropyLoss()
                                   lowerCamelCase_							=loss_fct(logits_item.view(-1,	self.num_labels						),	labels.view(-1						)						)
                            if total_loss is None:
                                   lowerCamelCase_							=loss
                            else:
                                   total_loss += loss * (ix + 1)
                            total_weights += ix + 1
                     lowerCamelCase_							=(total_loss / total_weights,) + outputs

              return outputs










'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_   :	Union[str, Any]							   = {
    """configuration_openai""": ["""OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """OpenAIGPTConfig"""],
    """tokenization_openai""": ["""OpenAIGPTTokenizer"""],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	Optional[int]							   = ["""OpenAIGPTTokenizerFast"""]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	int							   = [
         """OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """OpenAIGPTDoubleHeadsModel""",
         """OpenAIGPTForSequenceClassification""",
         """OpenAIGPTLMHeadModel""",
         """OpenAIGPTModel""",
         """OpenAIGPTPreTrainedModel""",
         """load_tf_weights_in_openai_gpt""",
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	Dict							   = [
         """TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """TFOpenAIGPTDoubleHeadsModel""",
         """TFOpenAIGPTForSequenceClassification""",
         """TFOpenAIGPTLMHeadModel""",
         """TFOpenAIGPTMainLayer""",
         """TFOpenAIGPTModel""",
         """TFOpenAIGPTPreTrainedModel""",
     ]


if TYPE_CHECKING:
     from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
     from .tokenization_openai import OpenAIGPTTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_openai_fast import OpenAIGPTTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_openai import (
              OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              OpenAIGPTDoubleHeadsModel,
              OpenAIGPTForSequenceClassification,
              OpenAIGPTLMHeadModel,
              OpenAIGPTModel,
              OpenAIGPTPreTrainedModel,
              load_tf_weights_in_openai_gpt,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_openai import (
              TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFOpenAIGPTDoubleHeadsModel,
              TFOpenAIGPTForSequenceClassification,
              TFOpenAIGPTLMHeadModel,
              TFOpenAIGPTMainLayer,
              TFOpenAIGPTModel,
              TFOpenAIGPTPreTrainedModel,
          )

else:
     import sys

     a_   :	List[Any]							   = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




