

import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


_SCREAMING_SNAKE_CASE							 =			logging.getLogger(__name__)


class      lowerCamelCase_ (			_lowerCAmelCase):




 def     a       (			self  ,  _UpperCAmelCase  ,  _UpperCAmelCase  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None							):
   if current_layer == 0:
     __UpperCAmelCase	:  Union[str, Any]		=     self.embedding_hidden_mapping_in(_UpperCAmelCase							)
   else:
     __UpperCAmelCase	:  List[Any]		=     hidden_states[0]

   __UpperCAmelCase	:  Tuple		=     int(self.config.num_hidden_layers / self.config.num_hidden_groups							)

   # Index of the hidden group
   __UpperCAmelCase	:  Optional[int]		=     int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)							)

   __UpperCAmelCase	:  Any		=     self.albert_layer_groups[group_idx](
       _UpperCAmelCase  ,  _UpperCAmelCase  ,  head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]  ,  )
   __UpperCAmelCase	:  Optional[int]		=     layer_group_output[0]

   return (hidden_states,)



@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''	,				_lowerCAmelCase	,				)
class      lowerCamelCase_ (			_lowerCAmelCase):




 def __init__(			self  ,  _UpperCAmelCase							):
   super().__init__(_UpperCAmelCase							)

   __UpperCAmelCase	:  Optional[int]		=     AlbertTransformerWithPabee(_UpperCAmelCase							)

   self.init_weights()
   __UpperCAmelCase	:  int		=     0
   __UpperCAmelCase	:  Optional[int]		=     0
   __UpperCAmelCase	:  Dict		=     0

   __UpperCAmelCase	:  Optional[int]		=     0




 def     a       (			self  ,  _UpperCAmelCase							):
   __UpperCAmelCase	:  int		=     threshold




 def     a       (			self  ,  _UpperCAmelCase							):
   __UpperCAmelCase	:  Union[str, Any]		=     patience




 def     a       (			self							):
   __UpperCAmelCase	:  List[Any]		=     0
   __UpperCAmelCase	:  Tuple		=     0




 def     a       (			self							):
   __UpperCAmelCase	:  List[str]		=     self.inference_layers_num / self.inference_instances_num
   __UpperCAmelCase	:  List[Any]		=     (
       F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
       F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
   )
   print(_UpperCAmelCase							)




 @add_start_docstrings_to_model_forward(_UpperCAmelCase							)
 def     a       (			self  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=False  ,  ):

   if input_ids is not None and inputs_embeds is not None:
     raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'							)
   elif input_ids is not None:
     __UpperCAmelCase	:  List[Any]		=     input_ids.size()
   elif inputs_embeds is not None:
     __UpperCAmelCase	:  Optional[Any]		=     inputs_embeds.size()[:-1]
   else:
     raise ValueError('You have to specify either input_ids or inputs_embeds'							)

   __UpperCAmelCase	:  Any		=     input_ids.device if input_ids is not None else inputs_embeds.device

   if attention_mask is None:
     __UpperCAmelCase	:  Any		=     torch.ones(_UpperCAmelCase  ,  device=_UpperCAmelCase							)
   if token_type_ids is None:
     __UpperCAmelCase	:  Dict		=     torch.zeros(_UpperCAmelCase  ,  dtype=torch.long  ,  device=_UpperCAmelCase							)

   __UpperCAmelCase	:  int		=     attention_mask.unsqueeze(1							).unsqueeze(2							)
   __UpperCAmelCase	:  List[Any]		=     extended_attention_mask.to(dtype=self.dtype							)  # fp16 compatibility
   __UpperCAmelCase	:  Dict		=     (1.0 - extended_attention_mask) * -1_0_0_0_0.0
   __UpperCAmelCase	:  Dict		=     self.get_head_mask(_UpperCAmelCase  ,  self.config.num_hidden_layers							)

   __UpperCAmelCase	:  Optional[Any]		=     self.embeddings(
       _UpperCAmelCase  ,  position_ids=_UpperCAmelCase  ,  token_type_ids=_UpperCAmelCase  ,  inputs_embeds=_UpperCAmelCase							)
   __UpperCAmelCase	:  int		=     embedding_output

   if self.training:
     __UpperCAmelCase	:  Dict		=     []
     for i in range(self.config.num_hidden_layers							):
       __UpperCAmelCase	:  Union[str, Any]		=     self.encoder.adaptive_forward(
           _UpperCAmelCase  ,  current_layer=_UpperCAmelCase  ,  attention_mask=_UpperCAmelCase  ,  head_mask=_UpperCAmelCase  ,  )

       __UpperCAmelCase	:  int		=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]							)							)
       __UpperCAmelCase	:  Optional[Any]		=     output_layers[i](output_dropout(_UpperCAmelCase							)							)
       res.append(_UpperCAmelCase							)
   elif self.patience == 0:  # Use all layers for inference
     __UpperCAmelCase	:  str		=     self.encoder(_UpperCAmelCase  ,  _UpperCAmelCase  ,  head_mask=_UpperCAmelCase							)
     __UpperCAmelCase	:  List[str]		=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]							)							)
     __UpperCAmelCase	:  Tuple		=     [output_layers[self.config.num_hidden_layers - 1](_UpperCAmelCase							)]
   else:
     __UpperCAmelCase	:  Tuple		=     0
     __UpperCAmelCase	:  List[Any]		=     None
     __UpperCAmelCase	:  Any		=     0
     for i in range(self.config.num_hidden_layers							):
       calculated_layer_num += 1
       __UpperCAmelCase	:  Union[str, Any]		=     self.encoder.adaptive_forward(
           _UpperCAmelCase  ,  current_layer=_UpperCAmelCase  ,  attention_mask=_UpperCAmelCase  ,  head_mask=_UpperCAmelCase  ,  )

       __UpperCAmelCase	:  List[str]		=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]							)							)
       __UpperCAmelCase	:  Tuple		=     output_layers[i](_UpperCAmelCase							)
       if regression:
         __UpperCAmelCase	:  Dict		=     logits.detach()
         if patient_result is not None:
           __UpperCAmelCase	:  Any		=     patient_result.detach()
         if (patient_result is not None) and torch.abs(patient_result - labels							) < self.regression_threshold:
           patient_counter += 1
         else:
           __UpperCAmelCase	:  Union[str, Any]		=     0
       else:
         __UpperCAmelCase	:  List[str]		=     logits.detach().argmax(dim=1							)
         if patient_result is not None:
           __UpperCAmelCase	:  List[Any]		=     patient_result.detach().argmax(dim=1							)
         if (patient_result is not None) and torch.all(labels.eq(_UpperCAmelCase							)							):
           patient_counter += 1
         else:
           __UpperCAmelCase	:  Optional[int]		=     0

       __UpperCAmelCase	:  Union[str, Any]		=     logits
       if patient_counter == self.patience:
         break
     __UpperCAmelCase	:  List[Any]		=     [patient_result]
     self.inference_layers_num += calculated_layer_num
     self.inference_instances_num += 1

   return res



@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''	,				_lowerCAmelCase	,				)
class      lowerCamelCase_ (			_lowerCAmelCase):




 def __init__(			self  ,  _UpperCAmelCase							):
   super().__init__(_UpperCAmelCase							)
   __UpperCAmelCase	:  str		=     config.num_labels

   __UpperCAmelCase	:  Any		=     AlbertModelWithPabee(_UpperCAmelCase							)
   __UpperCAmelCase	:  Tuple		=     nn.Dropout(config.classifier_dropout_prob							)
   __UpperCAmelCase	:  Optional[int]		=     nn.ModuleList(
       [nn.Linear(config.hidden_size  ,  self.config.num_labels							) for _ in range(config.num_hidden_layers							)]							)

   self.init_weights()




 @add_start_docstrings_to_model_forward(_UpperCAmelCase							)
 def     a       (			self  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  _UpperCAmelCase=None  ,  ):
   __UpperCAmelCase	:  Tuple		=     self.albert(
       input_ids=_UpperCAmelCase  ,  attention_mask=_UpperCAmelCase  ,  token_type_ids=_UpperCAmelCase  ,  position_ids=_UpperCAmelCase  ,  head_mask=_UpperCAmelCase  ,  inputs_embeds=_UpperCAmelCase  ,  output_dropout=self.dropout  ,  output_layers=self.classifiers  ,  regression=self.num_labels == 1  ,  )

   __UpperCAmelCase	:  List[Any]		=     (logits[-1],)

   if labels is not None:
     __UpperCAmelCase	:  Optional[int]		=     None
     __UpperCAmelCase	:  Dict		=     0
     for ix, logits_item in enumerate(_UpperCAmelCase							):
       if self.num_labels == 1:
         #  We are doing regression
         __UpperCAmelCase	:  List[Any]		=     MSELoss()
         __UpperCAmelCase	:  Any		=     loss_fct(logits_item.view(-1							)  ,  labels.view(-1							)							)
       else:
         __UpperCAmelCase	:  Tuple		=     CrossEntropyLoss()
         __UpperCAmelCase	:  List[Any]		=     loss_fct(logits_item.view(-1  ,  self.num_labels							)  ,  labels.view(-1							)							)
       if total_loss is None:
         __UpperCAmelCase	:  Union[str, Any]		=     loss
       else:
         total_loss += loss * (ix + 1)
       total_weights += ix + 1
     __UpperCAmelCase	:  List[Any]		=     (total_loss / total_weights,) + outputs

   return outputs






'''simple docstring'''




import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


__snake_case  : Dict		       =    500_000

__snake_case       ,       __snake_case  : List[str]		       =    os.path.split(__file__)
__snake_case  : Optional[Any]		       =    os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))





@get_duration
def 	_UpperCAmelCase       (				_UpperCamelCase       :      datasets.Dataset  )      -> Union[str, Any]:
    A_      =  dataset.select(range(0,  len(_UpperCamelCase  ),  2  )  )





@get_duration
def 	_UpperCAmelCase       (				_UpperCamelCase       :      datasets.Dataset  )      -> str:
    A_      =  dataset.sort('''numbers'''  )





@get_duration
def 	_UpperCAmelCase       (				_UpperCamelCase       :      datasets.Dataset  )      -> str:
    A_      =  dataset.shuffle()





@get_duration
def 	_UpperCAmelCase       (				_UpperCamelCase       :      datasets.Dataset  )      -> Optional[Any]:
    A_      =  dataset.train_test_split(0.1  )





@get_duration
def 	_UpperCAmelCase       (				_UpperCamelCase       :      datasets.Dataset,  _UpperCamelCase       :      Optional[int]=10  )      -> Dict:
    for shard_id in range(_UpperCamelCase  ):
        A_      =  dataset.shard(_UpperCamelCase,  _UpperCamelCase  )





def 	_UpperCAmelCase       (				)      -> Any:
    A_      =  {'''num examples''': SPEED_TEST_N_EXAMPLES}
    A_      =  (select, sort, shuffle, train_test_split, shard)
    with tempfile.TemporaryDirectory() as tmp_dir:
        print('''generating dataset'''  )
        A_      =  datasets.Features({'''text''': datasets.Value('''string'''  ), '''numbers''': datasets.Value('''float32'''  )}  )
        A_      =  generate_example_dataset(
            os.path.join(_UpperCamelCase,  '''dataset.arrow'''  ),  _UpperCamelCase,  num_examples=_UpperCamelCase  )
        print('''Functions'''  )
        for func in functions:
            print(func.__name__  )
            A_      =  func(_UpperCamelCase  )

    with open(_UpperCamelCase,  '''wb'''  ) as f:
        f.write(json.dumps(_UpperCamelCase  ).encode('''utf-8'''  )  )


if __name__ == "__main__":  # useful to run the profiler
  benchmark_indices_mapping()






