code: 
  
  
  
  """simple docstring"""
  from collections import OrderedDict
  from typing import Any, Mapping, Optional
  
  from ... import PreTrainedTokenizer
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
  from ...onnx.utils import compute_effective_axis_dimension
  from ...utils import TensorType, is_torch_available, logging
  
  
  _lowerCAmelCase					:int  =      logging.get_logger(__name__)
  
  _lowerCAmelCase					:int  =      {
      'google/bigbird-pegasus-large-arxiv': (
          'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
      ),
      'google/bigbird-pegasus-large-pubmed': (
          'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
      ),
      'google/bigbird-pegasus-large-bigpatent': (
          'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
      ),
      # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
  }
  class 							_UpperCAmelCase							(				lowerCamelCase__   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       a__             ='bigbird_pegasus'
       a__             =['past_key_values']
       a__             ={
           'num_attention_heads': 'encoder_attention_heads',
           'hidden_size': 'd_model',
           'attention_probs_dropout_prob': 'attention_dropout',
       }
  
  
  
  
  
       def __init__(      self ,			A=9_6_1_0_3 ,			A=4_0_9_6 ,			A=1_6 ,			A=4_0_9_6 ,			A=1_6 ,			A=1_6 ,			A=4_0_9_6 ,			A=1_6 ,			A=0.0 ,			A=0.0 ,			A=True ,			A=True ,			A="gelu_new" ,			A=1_0_2_4 ,			A=0.1 ,			A=0.0 ,			A=0.0 ,			A=0.02 ,			A=2 ,			A=0.0 ,			A=True ,			A=0 ,			A=2 ,			A=1 ,			A="block_sparse" ,			A=6_4 ,			A=3 ,			A=False ,			**A ,			)							->      List[str]:
           _UpperCAmelCase							:						Optional[int]       							=					vocab_size
           _UpperCAmelCase							:						Any       							=					max_position_embeddings
           _UpperCAmelCase							:						int       							=					d_model
           _UpperCAmelCase							:						Union[str, Any]       							=					encoder_ffn_dim
           _UpperCAmelCase							:						Tuple       							=					encoder_layers
           _UpperCAmelCase							:						Tuple       							=					encoder_attention_heads
           _UpperCAmelCase							:						str       							=					decoder_ffn_dim
           _UpperCAmelCase							:						int       							=					decoder_layers
           _UpperCAmelCase							:						Optional[int]       							=					decoder_attention_heads
           _UpperCAmelCase							:						Dict       							=					dropout
           _UpperCAmelCase							:						Optional[Any]       							=					attention_dropout
           _UpperCAmelCase							:						str       							=					activation_dropout
           _UpperCAmelCase							:						Union[str, Any]       							=					activation_function
           _UpperCAmelCase							:						int       							=					init_std
           _UpperCAmelCase							:						Union[str, Any]       							=					encoder_layerdrop
           _UpperCAmelCase							:						Optional[Any]       							=					decoder_layerdrop
           _UpperCAmelCase							:						Any       							=					classifier_dropout
           _UpperCAmelCase							:						List[Any]       							=					use_cache
           _UpperCAmelCase							:						List[str]       							=					encoder_layers
           _UpperCAmelCase							:						Union[str, Any]       							=					scale_embedding  # scale factor will be sqrt(d_model) if True
  
           # extra config
           _UpperCAmelCase							:						Optional[Any]       							=					attention_type
           _UpperCAmelCase							:						Optional[int]       							=					block_size
           _UpperCAmelCase							:						List[Any]       							=					num_random_blocks
           _UpperCAmelCase							:						int       							=					use_bias
  
           super().__init__(
               pad_token_id=lowercase_ ,			bos_token_id=lowercase_ ,			eos_token_id=lowercase_ ,			is_encoder_decoder=lowercase_ ,			decoder_start_token_id=lowercase_ ,			**lowercase_ ,			)
  
  
  
  class 							_UpperCAmelCase							(				lowerCamelCase__   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       @property
       def 				__lowerCAmelCase			(      self	)							->      Mapping[str, Mapping[int, str]]:
  
  
  
  
  
  
           if self.task in ["default", "seq2seq-lm"]:
               _UpperCAmelCase							:						List[str]       							=					OrderedDict(
                   [
                       ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                       ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                   ]	)
  
               if self.use_past:
                   _UpperCAmelCase							:						Optional[int]       							=					{0: '''batch'''}
                   _UpperCAmelCase							:						List[str]       							=					{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
               else:
                   _UpperCAmelCase							:						Union[str, Any]       							=					{0: '''batch''', 1: '''decoder_sequence'''}
                   _UpperCAmelCase							:						Dict       							=					{0: '''batch''', 1: '''decoder_sequence'''}
  
               if self.use_past:
                   self.fill_with_past_key_values_(lowercase_ ,			direction='''inputs'''	)
           elif self.task == "causal-lm":
               # TODO: figure this case out.
               _UpperCAmelCase							:						Optional[Any]       							=					OrderedDict(
                   [
                       ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                       ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                   ]	)
               if self.use_past:
                   _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[Any]       							=					self.num_layers
                   for i in range(lowercase_	):
                       _UpperCAmelCase							:						Optional[int]       							=					{0: '''batch''', 2: '''past_sequence + sequence'''}
                       _UpperCAmelCase							:						Union[str, Any]       							=					{0: '''batch''', 2: '''past_sequence + sequence'''}
           else:
               _UpperCAmelCase							:						List[Any]       							=					OrderedDict(
                   [
                       ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                       ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                       ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                       ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
                   ]	)
  
           return common_inputs
  
  
  
  
  
       @property
       def 				__lowerCAmelCase			(      self	)							->      Mapping[str, Mapping[int, str]]:
  
  
  
  
  
  
           if self.task in ["default", "seq2seq-lm"]:
               _UpperCAmelCase							:						str       							=					super().outputs
           else:
               _UpperCAmelCase							:						Union[str, Any]       							=					super(lowercase_ ,			self	).outputs
               if self.use_past:
                   _UpperCAmelCase							,       _UpperCAmelCase							:						Tuple       							=					self.num_layers
                   for i in range(lowercase_	):
                       _UpperCAmelCase							:						Union[str, Any]       							=					{0: '''batch''', 2: '''past_sequence + sequence'''}
                       _UpperCAmelCase							:						List[Any]       							=					{0: '''batch''', 2: '''past_sequence + sequence'''}
           return common_outputs
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A = -1 ,			A = -1 ,			A = False ,			A = None ,			)							->      Mapping[str, Any]:
           _UpperCAmelCase							:						Optional[Any]       							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
               lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_	)
  
           # Generate decoder inputs
           _UpperCAmelCase							:						Any       							=					seq_length if not self.use_past else 1
           _UpperCAmelCase							:						int       							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
               lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_	)
           _UpperCAmelCase							:						Optional[Any]       							=					{f'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
           _UpperCAmelCase							:						Union[str, Any]       							=					dict(**lowercase_ ,			**lowercase_	)
  
           if self.use_past:
               if not is_torch_available():
                   raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
               else:
                   import torch
               _UpperCAmelCase							,       _UpperCAmelCase							:						List[Any]       							=					common_inputs['''input_ids'''].shape
               _UpperCAmelCase							:						int       							=					common_inputs['''decoder_input_ids'''].shape[1]
               _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					self.num_attention_heads
               _UpperCAmelCase							:						Union[str, Any]       							=					(
                   batch,
                   num_encoder_attention_heads,
                   encoder_seq_length,
                   self._config.hidden_size // num_encoder_attention_heads,
               )
               _UpperCAmelCase							:						Union[str, Any]       							=					decoder_seq_length + 3
               _UpperCAmelCase							:						Optional[int]       							=					(
                   batch,
                   num_decoder_attention_heads,
                   decoder_past_length,
                   self._config.hidden_size // num_decoder_attention_heads,
               )
  
               _UpperCAmelCase							:						int       							=					torch.cat(
                   [common_inputs['''decoder_attention_mask'''], torch.ones(lowercase_ ,			lowercase_	)] ,			dim=1	)
  
               _UpperCAmelCase							:						Optional[int]       							=					[]
               # If the number of encoder and decoder layers are present in the model configuration, both are considered
               _UpperCAmelCase							,       _UpperCAmelCase							:						List[Any]       							=					self.num_layers
               _UpperCAmelCase							:						int       							=					min(lowercase_ ,			lowercase_	)
               _UpperCAmelCase							:						List[str]       							=					max(lowercase_ ,			lowercase_	) - min_num_layers
               _UpperCAmelCase							:						Tuple       							=					'''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''
  
               for _ in range(lowercase_	):
                   common_inputs["past_key_values"].append(
                       (
                           torch.zeros(lowercase_	),
                           torch.zeros(lowercase_	),
                           torch.zeros(lowercase_	),
                           torch.zeros(lowercase_	),
                       )	)
               # TODO: test this.
               _UpperCAmelCase							:						Optional[int]       							=					encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
               for _ in range(lowercase_ ,			lowercase_	):
                   common_inputs["past_key_values"].append((torch.zeros(lowercase_	), torch.zeros(lowercase_	))	)
           return common_inputs
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A = -1 ,			A = -1 ,			A = False ,			A = None ,			)							->      Mapping[str, Any]:
           _UpperCAmelCase							:						Optional[Any]       							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
               lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_	)
  
           if self.use_past:
               if not is_torch_available():
                   raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
               else:
                   import torch
               _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[Any]       							=					common_inputs['''input_ids'''].shape
               # Not using the same length for past_key_values
               _UpperCAmelCase							:						List[str]       							=					seqlen + 2
               _UpperCAmelCase							,       _UpperCAmelCase							:						List[Any]       							=					self.num_layers
               _UpperCAmelCase							,       _UpperCAmelCase							:						List[Any]       							=					self.num_attention_heads
               _UpperCAmelCase							:						int       							=					(
                   batch,
                   num_encoder_attention_heads,
                   past_key_values_length,
                   self._config.hidden_size // num_encoder_attention_heads,
               )
  
               _UpperCAmelCase							:						int       							=					common_inputs['''attention_mask'''].dtype
               _UpperCAmelCase							:						List[str]       							=					torch.cat(
                   [common_inputs['''attention_mask'''], torch.ones(lowercase_ ,			lowercase_ ,			dtype=lowercase_	)] ,			dim=1	)
               _UpperCAmelCase							:						List[Any]       							=					[
                   (torch.zeros(lowercase_	), torch.zeros(lowercase_	)) for _ in range(lowercase_	)
               ]
           return common_inputs
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A = -1 ,			A = -1 ,			A = False ,			A = None ,			)							->      Mapping[str, Any]:
           _UpperCAmelCase							:						List[Any]       							=					compute_effective_axis_dimension(
               lowercase_ ,			fixed_dimension=OnnxConfig.default_fixed_batch ,			num_token_to_add=0	)
  
           # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
           _UpperCAmelCase							:						int       							=					tokenizer.num_special_tokens_to_add(lowercase_	)
           _UpperCAmelCase							:						Dict       							=					compute_effective_axis_dimension(
               lowercase_ ,			fixed_dimension=OnnxConfig.default_fixed_sequence ,			num_token_to_add=lowercase_	)
  
           # Generate dummy inputs according to compute batch and sequence
           _UpperCAmelCase							:						List[str]       							=					[''' '''.join([tokenizer.unk_token]	) * seq_length] * batch_size
           _UpperCAmelCase							:						int       							=					dict(tokenizer(lowercase_ ,			return_tensors=lowercase_	)	)
           return common_inputs
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A = -1 ,			A = -1 ,			A = False ,			A = None ,			)							->      Mapping[str, Any]:
  
  
  
  
  
  
           if self.task in ["default", "seq2seq-lm"]:
               _UpperCAmelCase							:						List[Any]       							=					self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                   lowercase_ ,			batch_size=lowercase_ ,			seq_length=lowercase_ ,			is_pair=lowercase_ ,			framework=lowercase_	)
  
           elif self.task == "causal-lm":
               _UpperCAmelCase							:						List[Any]       							=					self._generate_dummy_inputs_for_causal_lm(
                   lowercase_ ,			batch_size=lowercase_ ,			seq_length=lowercase_ ,			is_pair=lowercase_ ,			framework=lowercase_	)
           else:
               _UpperCAmelCase							:						str       							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                   lowercase_ ,			batch_size=lowercase_ ,			seq_length=lowercase_ ,			is_pair=lowercase_ ,			framework=lowercase_	)
  
           return common_inputs
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A ,			A ,			A	)							->      Optional[Any]:
  
  
  
  
  
  
           if self.task in ["default", "seq2seq-lm"]:
               _UpperCAmelCase							:						int       							=					super()._flatten_past_key_values_(lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_	)
           else:
               _UpperCAmelCase							:						Optional[Any]       							=					super(lowercase_ ,			self	)._flatten_past_key_values_(
                   lowercase_ ,			lowercase_ ,			lowercase_ ,			lowercase_	)
  
  
  
style_context: 
  
  
  
  """simple docstring"""
  import logging
  
  import torch
  from torch import nn
  from torch.nn import CrossEntropyLoss, MSELoss
  
  from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
  from transformers.models.albert.modeling_albert import (
      ALBERT_INPUTS_DOCSTRING,
      ALBERT_START_DOCSTRING,
      AlbertModel,
      AlbertPreTrainedModel,
      AlbertTransformer,
  )
  
  
  _lowerCAmelCase					:Union[str, Any]  =      logging.getLogger(__name__)
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A ,			A=None ,			A=None	)							->      Tuple:
           if current_layer == 0:
               _UpperCAmelCase							:						Optional[Any]       							=					self.embedding_hidden_mapping_in(A	)
           else:
               _UpperCAmelCase							:						Union[str, Any]       							=					hidden_states[0]
  
           _UpperCAmelCase							:						Dict       							=					int(self.config.num_hidden_layers / self.config.num_hidden_groups	)
  
           # Index of the hidden group
           _UpperCAmelCase							:						Dict       							=					int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)	)
  
           _UpperCAmelCase							:						List[Any]       							=					self.albert_layer_groups[group_idx](
               A ,			A ,			head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group] ,			)
           _UpperCAmelCase							:						str       							=					layer_group_output[0]
  
           return (hidden_states,)
  
  
  
  @add_start_docstrings(
      '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''	,a	,)
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A	)							->      str:
           super().__init__(A	)
  
           _UpperCAmelCase							:						Dict       							=					AlbertTransformerWithPabee(A	)
  
           self.init_weights()
           _UpperCAmelCase							:						str       							=					0
           _UpperCAmelCase							:						Union[str, Any]       							=					0
           _UpperCAmelCase							:						Dict       							=					0
  
           _UpperCAmelCase							:						str       							=					0
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A	)							->      Optional[int]:
           _UpperCAmelCase							:						List[Any]       							=					threshold
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A	)							->      Optional[Any]:
           _UpperCAmelCase							:						int       							=					patience
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Any:
           _UpperCAmelCase							:						Dict       							=					0
           _UpperCAmelCase							:						List[str]       							=					0
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      List[Any]:
           _UpperCAmelCase							:						Union[str, Any]       							=					self.inference_layers_num / self.inference_instances_num
           _UpperCAmelCase							:						str       							=					(
               f'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
               f' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
           )
           print(A	)
  
  
  
  
  
       @add_start_docstrings_to_model_forward(A	)
       def 				__lowerCAmelCase			(      self ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=False ,			)							->      int:
  
           if input_ids is not None and inputs_embeds is not None:
               raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''	)
           elif input_ids is not None:
               _UpperCAmelCase							:						Optional[Any]       							=					input_ids.size()
           elif inputs_embeds is not None:
               _UpperCAmelCase							:						Optional[Any]       							=					inputs_embeds.size()[:-1]
           else:
               raise ValueError('''You have to specify either input_ids or inputs_embeds'''	)
  
           _UpperCAmelCase							:						Tuple       							=					input_ids.device if input_ids is not None else inputs_embeds.device
  
           if attention_mask is None:
               _UpperCAmelCase							:						Union[str, Any]       							=					torch.ones(A ,			device=A	)
           if token_type_ids is None:
               _UpperCAmelCase							:						Tuple       							=					torch.zeros(A ,			dtype=torch.long ,			device=A	)
  
           _UpperCAmelCase							:						Optional[Any]       							=					attention_mask.unsqueeze(1	).unsqueeze(2	)
           _UpperCAmelCase							:						Union[str, Any]       							=					extended_attention_mask.to(dtype=self.dtype	)  # fp16 compatibility
           _UpperCAmelCase							:						Dict       							=					(1.0 - extended_attention_mask) * -10_000.0
           _UpperCAmelCase							:						Any       							=					self.get_head_mask(A ,			self.config.num_hidden_layers	)
  
           _UpperCAmelCase							:						int       							=					self.embeddings(
               A ,			position_ids=A ,			token_type_ids=A ,			inputs_embeds=A	)
           _UpperCAmelCase							:						Any       							=					embedding_output
  
           if self.training:
               _UpperCAmelCase							:						Optional[Any]       							=					[]
               for i in range(self.config.num_hidden_layers	):
                   _UpperCAmelCase							:						Union[str, Any]       							=					self.encoder.adaptive_forward(
                       A ,			current_layer=A ,			attention_mask=A ,			head_mask=A ,			)
  
                   _UpperCAmelCase							:						List[str]       							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]	)	)
                   _UpperCAmelCase							:						Dict       							=					output_layers[i](output_dropout(A	)	)
                   res.append(A	)
           elif self.patience == 0:  # Use all layers for inference
               _UpperCAmelCase							:						Dict       							=					self.encoder(A ,			A ,			head_mask=A	)
               _UpperCAmelCase							:						Tuple       							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]	)	)
               _UpperCAmelCase							:						Optional[int]       							=					[output_layers[self.config.num_hidden_layers - 1](A	)]
           else:
               _UpperCAmelCase							:						List[Any]       							=					0
               _UpperCAmelCase							:						int       							=					None
               _UpperCAmelCase							:						int       							=					0
               for i in range(self.config.num_hidden_layers	):
                   calculated_layer_num += 1
                   _UpperCAmelCase							:						Tuple       							=					self.encoder.adaptive_forward(
                       A ,			current_layer=A ,			attention_mask=A ,			head_mask=A ,			)
  
                   _UpperCAmelCase							:						Any       							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]	)	)
                   _UpperCAmelCase							:						Optional[Any]       							=					output_layers[i](A	)
                   if regression:
                       _UpperCAmelCase							:						str       							=					logits.detach()
                       if patient_result is not None:
                           _UpperCAmelCase							:						str       							=					patient_result.detach()
                       if (patient_result is not None) and torch.abs(patient_result - labels	) < self.regression_threshold:
                           patient_counter += 1
                       else:
                           _UpperCAmelCase							:						List[Any]       							=					0
                   else:
                       _UpperCAmelCase							:						List[str]       							=					logits.detach().argmax(dim=1	)
                       if patient_result is not None:
                           _UpperCAmelCase							:						Optional[Any]       							=					patient_result.detach().argmax(dim=1	)
                       if (patient_result is not None) and torch.all(labels.eq(A	)	):
                           patient_counter += 1
                       else:
                           _UpperCAmelCase							:						Tuple       							=					0
  
                   _UpperCAmelCase							:						Any       							=					logits
                   if patient_counter == self.patience:
                       break
               _UpperCAmelCase							:						List[str]       							=					[patient_result]
               self.inference_layers_num += calculated_layer_num
               self.inference_instances_num += 1
  
           return res
  
  
  
  @add_start_docstrings(
      '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
      the pooled output) e.g. for GLUE tasks. '''	,a	,)
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A	)							->      Optional[int]:
           super().__init__(A	)
           _UpperCAmelCase							:						List[Any]       							=					config.num_labels
  
           _UpperCAmelCase							:						Optional[int]       							=					AlbertModelWithPabee(A	)
           _UpperCAmelCase							:						List[str]       							=					nn.Dropout(config.classifier_dropout_prob	)
           _UpperCAmelCase							:						Optional[Any]       							=					nn.ModuleList(
               [nn.Linear(config.hidden_size ,			self.config.num_labels	) for _ in range(config.num_hidden_layers	)]	)
  
           self.init_weights()
  
  
  
  
  
       @add_start_docstrings_to_model_forward(A	)
       def 				__lowerCAmelCase			(      self ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			)							->      List[str]:
           _UpperCAmelCase							:						Optional[Any]       							=					self.albert(
               input_ids=A ,			attention_mask=A ,			token_type_ids=A ,			position_ids=A ,			head_mask=A ,			inputs_embeds=A ,			output_dropout=self.dropout ,			output_layers=self.classifiers ,			regression=self.num_labels == 1 ,			)
  
           _UpperCAmelCase							:						int       							=					(logits[-1],)
  
           if labels is not None:
               _UpperCAmelCase							:						int       							=					None
               _UpperCAmelCase							:						Tuple       							=					0
               for ix, logits_item in enumerate(A	):
                   if self.num_labels == 1:
                       #  We are doing regression
                       _UpperCAmelCase							:						Dict       							=					MSELoss()
                       _UpperCAmelCase							:						Optional[int]       							=					loss_fct(logits_item.view(-1	) ,			labels.view(-1	)	)
                   else:
                       _UpperCAmelCase							:						List[str]       							=					CrossEntropyLoss()
                       _UpperCAmelCase							:						Any       							=					loss_fct(logits_item.view(-1 ,			self.num_labels	) ,			labels.view(-1	)	)
                   if total_loss is None:
                       _UpperCAmelCase							:						Optional[Any]       							=					loss
                   else:
                       total_loss += loss * (ix + 1)
                   total_weights += ix + 1
               _UpperCAmelCase							:						Any       							=					(total_loss / total_weights,) + outputs
  
           return outputs
  
  
  