code: 
  
  import logging
  
  import torch
  from torch import nn
  from torch.nn import CrossEntropyLoss, MSELoss
  
  from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
  from transformers.models.albert.modeling_albert import (
      ALBERT_INPUTS_DOCSTRING,
      ALBERT_START_DOCSTRING,
      AlbertModel,
      AlbertPreTrainedModel,
      AlbertTransformer,
  )
  
  
  __A   :     int      =			logging.getLogger(__name__)
  
  
  
  
  
  
  class    __A				(							A__ ):
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Any    ,				UpperCAmelCase_			:    Any    ,				UpperCAmelCase_			:    int    ,				UpperCAmelCase_			:    Optional[Any]=None    ,				UpperCAmelCase_			:    str=None    ):
          if current_layer == 0:
              lowerCAmelCase    			=      self.embedding_hidden_mapping_in(_lowerCAmelCase    )
          else:
              lowerCAmelCase    			=      hidden_states[0]
  
          lowerCAmelCase    			=      int(self.config.num_hidden_layers / self.config.num_hidden_groups    )
  
          # Index of the hidden group
          lowerCAmelCase    			=      int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)    )
  
          lowerCAmelCase    			=      self.albert_layer_groups[group_idx](
              _lowerCAmelCase    ,				_lowerCAmelCase    ,				head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]    ,				)
          lowerCAmelCase    			=      layer_group_output[0]
  
          return (hidden_states,)
  
  
  
  
  
  
  @add_start_docstrings(
      "The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."						,						A__						,						)
  class    __A				(							A__ ):
  
  
  
  
  
  
      def __init__(	self			:    Union[str, Any]    ,				UpperCAmelCase_			:    Optional[int]    ):
          super().__init__(_lowerCAmelCase    )
  
          lowerCAmelCase    			=      AlbertTransformerWithPabee(_lowerCAmelCase    )
  
          self.init_weights()
          lowerCAmelCase    			=      0
          lowerCAmelCase    			=      0
          lowerCAmelCase    			=      0
  
          lowerCAmelCase    			=      0
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Any    ,				UpperCAmelCase_			:    Tuple    ):
          lowerCAmelCase    			=      threshold
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Tuple    ,				UpperCAmelCase_			:    List[Any]    ):
          lowerCAmelCase    			=      patience
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          lowerCAmelCase    			=      0
          lowerCAmelCase    			=      0
  
  
  
  
  
  
      def 	lowercase__ (	self			:    str    ):
          lowerCAmelCase    			=      self.inference_layers_num / self.inference_instances_num
          lowerCAmelCase    			=      (
              f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
              f" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
          )
          print(_lowerCAmelCase    )
  
  
  
  
  
  
      @add_start_docstrings_to_model_forward(_lowerCAmelCase    )
      def 	lowercase__ (	self			:    Dict    ,				UpperCAmelCase_			:    int=None    ,				UpperCAmelCase_			:    str=None    ,				UpperCAmelCase_			:    Optional[int]=None    ,				UpperCAmelCase_			:    Union[str, Any]=None    ,				UpperCAmelCase_			:    List[str]=None    ,				UpperCAmelCase_			:    Optional[int]=None    ,				UpperCAmelCase_			:    Union[str, Any]=None    ,				UpperCAmelCase_			:    Tuple=None    ,				UpperCAmelCase_			:    List[Any]=False    ,				):
  
          if input_ids is not None and inputs_embeds is not None:
              raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'    )
          elif input_ids is not None:
              lowerCAmelCase    			=      input_ids.size()
          elif inputs_embeds is not None:
              lowerCAmelCase    			=      inputs_embeds.size()[:-1]
          else:
              raise ValueError('You have to specify either input_ids or inputs_embeds'    )
  
          lowerCAmelCase    			=      input_ids.device if input_ids is not None else inputs_embeds.device
  
          if attention_mask is None:
              lowerCAmelCase    			=      torch.ones(_lowerCAmelCase    ,				device=_lowerCAmelCase    )
          if token_type_ids is None:
              lowerCAmelCase    			=      torch.zeros(_lowerCAmelCase    ,				dtype=torch.long    ,				device=_lowerCAmelCase    )
  
          lowerCAmelCase    			=      attention_mask.unsqueeze(1    ).unsqueeze(2    )
          lowerCAmelCase    			=      extended_attention_mask.to(dtype=self.dtype    )  # fp16 compatibility
          lowerCAmelCase    			=      (1.0 - extended_attention_mask) * -10000.0
          lowerCAmelCase    			=      self.get_head_mask(_lowerCAmelCase    ,				self.config.num_hidden_layers    )
  
          lowerCAmelCase    			=      self.embeddings(
              _lowerCAmelCase    ,				position_ids=_lowerCAmelCase    ,				token_type_ids=_lowerCAmelCase    ,				inputs_embeds=_lowerCAmelCase    )
          lowerCAmelCase    			=      embedding_output
  
          if self.training:
              lowerCAmelCase    			=      []
              for i in range(self.config.num_hidden_layers    ):
                  lowerCAmelCase    			=      self.encoder.adaptive_forward(
                      _lowerCAmelCase    ,				current_layer=_lowerCAmelCase    ,				attention_mask=_lowerCAmelCase    ,				head_mask=_lowerCAmelCase    ,				)
  
                  lowerCAmelCase    			=      self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
                  lowerCAmelCase    			=      output_layers[i](output_dropout(_lowerCAmelCase    )    )
                  res.append(_lowerCAmelCase    )
          elif self.patience == 0:  # Use all layers for inference
              lowerCAmelCase    			=      self.encoder(_lowerCAmelCase    ,				_lowerCAmelCase    ,				head_mask=_lowerCAmelCase    )
              lowerCAmelCase    			=      self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
              lowerCAmelCase    			=      [output_layers[self.config.num_hidden_layers - 1](_lowerCAmelCase    )]
          else:
              lowerCAmelCase    			=      0
              lowerCAmelCase    			=      None
              lowerCAmelCase    			=      0
              for i in range(self.config.num_hidden_layers    ):
                  calculated_layer_num += 1
                  lowerCAmelCase    			=      self.encoder.adaptive_forward(
                      _lowerCAmelCase    ,				current_layer=_lowerCAmelCase    ,				attention_mask=_lowerCAmelCase    ,				head_mask=_lowerCAmelCase    ,				)
  
                  lowerCAmelCase    			=      self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
                  lowerCAmelCase    			=      output_layers[i](_lowerCAmelCase    )
                  if regression:
                      lowerCAmelCase    			=      logits.detach()
                      if patient_result is not None:
                          lowerCAmelCase    			=      patient_result.detach()
                      if (patient_result is not None) and torch.abs(patient_result - labels    ) < self.regression_threshold:
                          patient_counter += 1
                      else:
                          lowerCAmelCase    			=      0
                  else:
                      lowerCAmelCase    			=      logits.detach().argmax(dim=1    )
                      if patient_result is not None:
                          lowerCAmelCase    			=      patient_result.detach().argmax(dim=1    )
                      if (patient_result is not None) and torch.all(labels.eq(_lowerCAmelCase    )    ):
                          patient_counter += 1
                      else:
                          lowerCAmelCase    			=      0
  
                  lowerCAmelCase    			=      logits
                  if patient_counter == self.patience:
                      break
              lowerCAmelCase    			=      [patient_result]
              self.inference_layers_num += calculated_layer_num
              self.inference_instances_num += 1
  
          return res
  
  
  
  
  
  
  @add_start_docstrings(
      "Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "						,						A__						,						)
  class    __A				(							A__ ):
  
  
  
  
  
  
      def __init__(	self			:    Optional[Any]    ,				UpperCAmelCase_			:    str    ):
          super().__init__(_lowerCAmelCase    )
          lowerCAmelCase    			=      config.num_labels
  
          lowerCAmelCase    			=      AlbertModelWithPabee(_lowerCAmelCase    )
          lowerCAmelCase    			=      nn.Dropout(config.classifier_dropout_prob    )
          lowerCAmelCase    			=      nn.ModuleList(
              [nn.Linear(config.hidden_size    ,				self.config.num_labels    ) for _ in range(config.num_hidden_layers    )]    )
  
          self.init_weights()
  
  
  
  
  
  
      @add_start_docstrings_to_model_forward(_lowerCAmelCase    )
      def 	lowercase__ (	self			:    int    ,				UpperCAmelCase_			:    Any=None    ,				UpperCAmelCase_			:    Optional[int]=None    ,				UpperCAmelCase_			:    str=None    ,				UpperCAmelCase_			:    Optional[int]=None    ,				UpperCAmelCase_			:    int=None    ,				UpperCAmelCase_			:    Union[str, Any]=None    ,				UpperCAmelCase_			:    Tuple=None    ,				):
          lowerCAmelCase    			=      self.albert(
              input_ids=_lowerCAmelCase    ,				attention_mask=_lowerCAmelCase    ,				token_type_ids=_lowerCAmelCase    ,				position_ids=_lowerCAmelCase    ,				head_mask=_lowerCAmelCase    ,				inputs_embeds=_lowerCAmelCase    ,				output_dropout=self.dropout    ,				output_layers=self.classifiers    ,				regression=self.num_labels == 1    ,				)
  
          lowerCAmelCase    			=      (logits[-1],)
  
          if labels is not None:
              lowerCAmelCase    			=      None
              lowerCAmelCase    			=      0
              for ix, logits_item in enumerate(_lowerCAmelCase    ):
                  if self.num_labels == 1:
                      #  We are doing regression
                      lowerCAmelCase    			=      MSELoss()
                      lowerCAmelCase    			=      loss_fct(logits_item.view(-1    )    ,				labels.view(-1    )    )
                  else:
                      lowerCAmelCase    			=      CrossEntropyLoss()
                      lowerCAmelCase    			=      loss_fct(logits_item.view(-1    ,				self.num_labels    )    ,				labels.view(-1    )    )
                  if total_loss is None:
                      lowerCAmelCase    			=      loss
                  else:
                      total_loss += loss * (ix + 1)
                  total_weights += ix + 1
              lowerCAmelCase    			=      (total_loss / total_weights,) + outputs
  
          return outputs
  
  
  
  
  
  
style_context: 
  
  __A   :     List[Any]      =			'''
  # Installation de Transformers
  ! pip install transformers datasets
  # Pour installer à partir du code source au lieu de la dernière version, commentez la commande ci-dessus et décommentez la suivante.
  # ! pip install git+https://github.com/huggingface/transformers.git
  '''
  
  __A   :     Dict      =			[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
  __A   :     Tuple      =			{
      '''{processor_class}''': '''FakeProcessorClass''',
      '''{model_class}''': '''FakeModelClass''',
      '''{object_class}''': '''FakeObjectClass''',
  }
  
  
  
  
  
  