code: 
  
  
  import logging
  import os
  import sys
  from dataclasses import dataclass, field
  from typing import Optional
  
  import transformers
  from transformers import (
      AutoConfig,
      AutoModelForQuestionAnswering,
      AutoTokenizer,
      DataCollatorWithPadding,
      HfArgumentParser,
      SquadDataset,
      Trainer,
      TrainingArguments,
  )
  from transformers import SquadDataTrainingArguments as DataTrainingArguments
  from transformers.trainer_utils import is_main_process
  
  
  SCREAMING_SNAKE_CASE__				:    Tuple         =    logging.getLogger(__name__)
  
  
  @dataclass
  class 		_SCREAMING_SNAKE_CASE    :
       __SCREAMING_SNAKE_CASE    				=						field(
           metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}    )
       __SCREAMING_SNAKE_CASE    				=						field(
           default=A       ,  metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}    )
       __SCREAMING_SNAKE_CASE    				=						field(
           default=A       ,  metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}    )
       __SCREAMING_SNAKE_CASE    				=						field(default=A       ,  metadata={'''help''': '''Set this flag to use fast tokenization.'''}    )
       # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
       # or just modify its tokenizer_config.json.
       __SCREAMING_SNAKE_CASE    				=						field(
           default=A       ,  metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}       ,  )
  
  
  
  
  def       a__						(		):
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.
  
    _UpperCAmelCase		:       str       =    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)    )
  
    if len(sys.argv    ) == 2 and sys.argv[1].endswith(""".json"""    ):
      # If we pass only one argument to the script and it's the path to a json file,
      # let's parse it to get our arguments.
      _UpperCAmelCase,_UpperCAmelCase,_UpperCAmelCase		:       Any       =    parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]    )    )
    else:
      _UpperCAmelCase,_UpperCAmelCase,_UpperCAmelCase		:       Union[str, Any]       =    parser.parse_args_into_dataclasses()
  
    if (
        os.path.exists(training_args.output_dir    )
        and os.listdir(training_args.output_dir    )
        and training_args.do_train
        and not training_args.overwrite_output_dir
    ):
      raise ValueError(
          f'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
          """ --overwrite_output_dir to overcome."""    )
  
    # Setup logging
    logging.basicConfig(
        format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""	,      datefmt="""%m/%d/%Y %H:%M:%S"""	,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN	,      )
    logger.warning(
        """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""	,      training_args.local_rank	,      training_args.device	,      training_args.n_gpu	,      bool(training_args.local_rank != -1    )	,      training_args.fpaa	,      )
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank    ):
      transformers.utils.logging.set_verbosity_info()
      transformers.utils.logging.enable_default_handler()
      transformers.utils.logging.enable_explicit_format()
    logger.info("""Training/evaluation parameters %s"""	,      snake_case__    )
  
    # Prepare Question-Answering task
    # Load pretrained model and tokenizer
    #
    # Distributed training:
    # The .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.
  
    _UpperCAmelCase		:       Union[str, Any]       =    AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path	,      cache_dir=model_args.cache_dir	,      )
    _UpperCAmelCase		:       List[str]       =    AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path	,      cache_dir=model_args.cache_dir	,      use_fast=snake_case__	,      )
    _UpperCAmelCase		:       Any       =    AutoModelForQuestionAnswering.from_pretrained(
        model_args.model_name_or_path	,      from_tf=bool(""".ckpt""" in model_args.model_name_or_path    )	,      config=snake_case__	,      cache_dir=model_args.cache_dir	,      )
  
    # Get datasets
    _UpperCAmelCase		:       Optional[Any]       =    hasattr(model.config	,      """lang2id"""    )
    _UpperCAmelCase		:       Tuple       =    (
        SquadDataset(
            snake_case__	,      tokenizer=snake_case__	,      is_language_sensitive=snake_case__	,      cache_dir=model_args.cache_dir    )
        if training_args.do_train
        else None
    )
    _UpperCAmelCase		:       int       =    (
        SquadDataset(
            snake_case__	,      tokenizer=snake_case__	,      mode="""dev"""	,      is_language_sensitive=snake_case__	,      cache_dir=model_args.cache_dir	,      )
        if training_args.do_eval
        else None
    )
  
    # Data collator
    _UpperCAmelCase		:       Dict       =    DataCollatorWithPadding(snake_case__	,      pad_to_multiple_of=8    ) if training_args.fpaa else None
  
    # Initialize our Trainer
    _UpperCAmelCase		:       Any       =    Trainer(
        model=snake_case__	,      args=snake_case__	,      train_dataset=snake_case__	,      eval_dataset=snake_case__	,      data_collator=snake_case__	,      )
  
    # Training
    if training_args.do_train:
      trainer.train(
          model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path    ) else None    )
      trainer.save_model()
      # For convenience, we also re-save the tokenizer to the same directory,
      # so that you can share your model easily on huggingface.co/models =)
      if trainer.is_world_master():
        tokenizer.save_pretrained(training_args.output_dir    )
  
  
  
  
  def       a__						(		snake_case__				:       Any    ):
    # For xla_spawn (TPUs)
    main()
  
  
  if __name__ == "__main__":
   main()
  
  
  
  
  
  
style_context: 
  
  
  import logging
  
  import torch
  from torch import nn
  from torch.nn import CrossEntropyLoss, MSELoss
  
  from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
  from transformers.models.albert.modeling_albert import (
      ALBERT_INPUTS_DOCSTRING,
      ALBERT_START_DOCSTRING,
      AlbertModel,
      AlbertPreTrainedModel,
      AlbertTransformer,
  )
  
  
  SCREAMING_SNAKE_CASE__				:    Dict         =    logging.getLogger(__name__)
  
  
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_=None       ,				A_=None						):
         if current_layer == 0:
           _UpperCAmelCase		:       Any       =    self.embedding_hidden_mapping_in(A_						)
         else:
           _UpperCAmelCase		:       Optional[int]       =    hidden_states[0]
  
         _UpperCAmelCase		:       Tuple       =    int(self.config.num_hidden_layers / self.config.num_hidden_groups						)
  
         # Index of the hidden group
         _UpperCAmelCase		:       Union[str, Any]       =    int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)						)
  
         _UpperCAmelCase		:       int       =    self.albert_layer_groups[group_idx](
             A_       ,				A_       ,				head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]       ,				)
         _UpperCAmelCase		:       Dict       =    layer_group_output[0]
  
         return (hidden_states,)
  
  
  @add_start_docstrings(
      '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''       ,  A       ,  )
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
  
  
       def __init__(     self       ,				A_						):
         super().__init__(A_						)
  
         _UpperCAmelCase		:       List[Any]       =    AlbertTransformerWithPabee(A_						)
  
         self.init_weights()
         _UpperCAmelCase		:       Tuple       =    0
         _UpperCAmelCase		:       Optional[Any]       =    0
         _UpperCAmelCase		:       int       =    0
  
         _UpperCAmelCase		:       Dict       =    0
  
  
       def 					__snake_case(     self       ,				A_						):
         _UpperCAmelCase		:       str       =    threshold
  
  
       def 					__snake_case(     self       ,				A_						):
         _UpperCAmelCase		:       Optional[Any]       =    patience
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Tuple       =    0
         _UpperCAmelCase		:       Tuple       =    0
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Dict       =    self.inference_layers_num / self.inference_instances_num
         _UpperCAmelCase		:       Optional[int]       =    (
             F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
             F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
         )
         print(A_						)
  
  
  
  
       @add_start_docstrings_to_model_forward(A_						)
       def 					__snake_case(     self       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=False       ,				):
  
         if input_ids is not None and inputs_embeds is not None:
           raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""						)
         elif input_ids is not None:
           _UpperCAmelCase		:       List[str]       =    input_ids.size()
         elif inputs_embeds is not None:
           _UpperCAmelCase		:       Optional[Any]       =    inputs_embeds.size()[:-1]
         else:
           raise ValueError("""You have to specify either input_ids or inputs_embeds"""						)
  
         _UpperCAmelCase		:       List[str]       =    input_ids.device if input_ids is not None else inputs_embeds.device
  
         if attention_mask is None:
           _UpperCAmelCase		:       Any       =    torch.ones(A_       ,				device=A_						)
         if token_type_ids is None:
           _UpperCAmelCase		:       List[str]       =    torch.zeros(A_       ,				dtype=torch.long       ,				device=A_						)
  
         _UpperCAmelCase		:       str       =    attention_mask.unsqueeze(1						).unsqueeze(2						)
         _UpperCAmelCase		:       Tuple       =    extended_attention_mask.to(dtype=self.dtype						)  # fp16 compatibility
         _UpperCAmelCase		:       Tuple       =    (1.0 - extended_attention_mask) * -1_0_0_0_0.0
         _UpperCAmelCase		:       Union[str, Any]       =    self.get_head_mask(A_       ,				self.config.num_hidden_layers						)
  
         _UpperCAmelCase		:       Dict       =    self.embeddings(
             A_       ,				position_ids=A_       ,				token_type_ids=A_       ,				inputs_embeds=A_						)
         _UpperCAmelCase		:       str       =    embedding_output
  
         if self.training:
           _UpperCAmelCase		:       Any       =    []
           for i in range(self.config.num_hidden_layers						):
             _UpperCAmelCase		:       Tuple       =    self.encoder.adaptive_forward(
                 A_       ,				current_layer=A_       ,				attention_mask=A_       ,				head_mask=A_       ,				)
  
             _UpperCAmelCase		:       List[Any]       =    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
             _UpperCAmelCase		:       List[str]       =    output_layers[i](output_dropout(A_						)						)
             res.append(A_						)
         elif self.patience == 0:  # Use all layers for inference
           _UpperCAmelCase		:       Optional[int]       =    self.encoder(A_       ,				A_       ,				head_mask=A_						)
           _UpperCAmelCase		:       Optional[int]       =    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
           _UpperCAmelCase		:       List[Any]       =    [output_layers[self.config.num_hidden_layers - 1](A_						)]
         else:
           _UpperCAmelCase		:       str       =    0
           _UpperCAmelCase		:       str       =    None
           _UpperCAmelCase		:       str       =    0
           for i in range(self.config.num_hidden_layers						):
             calculated_layer_num += 1
             _UpperCAmelCase		:       int       =    self.encoder.adaptive_forward(
                 A_       ,				current_layer=A_       ,				attention_mask=A_       ,				head_mask=A_       ,				)
  
             _UpperCAmelCase		:       Tuple       =    self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
             _UpperCAmelCase		:       List[str]       =    output_layers[i](A_						)
             if regression:
               _UpperCAmelCase		:       Optional[Any]       =    logits.detach()
               if patient_result is not None:
                 _UpperCAmelCase		:       str       =    patient_result.detach()
               if (patient_result is not None) and torch.abs(patient_result - labels						) < self.regression_threshold:
                 patient_counter += 1
               else:
                 _UpperCAmelCase		:       Union[str, Any]       =    0
             else:
               _UpperCAmelCase		:       Dict       =    logits.detach().argmax(dim=1						)
               if patient_result is not None:
                 _UpperCAmelCase		:       Optional[Any]       =    patient_result.detach().argmax(dim=1						)
               if (patient_result is not None) and torch.all(labels.eq(A_						)						):
                 patient_counter += 1
               else:
                 _UpperCAmelCase		:       Dict       =    0
  
             _UpperCAmelCase		:       int       =    logits
             if patient_counter == self.patience:
               break
           _UpperCAmelCase		:       Tuple       =    [patient_result]
           self.inference_layers_num += calculated_layer_num
           self.inference_instances_num += 1
  
         return res
  
  
  @add_start_docstrings(
      '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
      the pooled output) e.g. for GLUE tasks. '''       ,  A       ,  )
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
  
  
       def __init__(     self       ,				A_						):
         super().__init__(A_						)
         _UpperCAmelCase		:       List[str]       =    config.num_labels
  
         _UpperCAmelCase		:       List[Any]       =    AlbertModelWithPabee(A_						)
         _UpperCAmelCase		:       Optional[int]       =    nn.Dropout(config.classifier_dropout_prob						)
         _UpperCAmelCase		:       Tuple       =    nn.ModuleList(
             [nn.Linear(config.hidden_size       ,				self.config.num_labels						) for _ in range(config.num_hidden_layers						)]						)
  
         self.init_weights()
  
  
  
  
       @add_start_docstrings_to_model_forward(A_						)
       def 					__snake_case(     self       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				A_=None       ,				):
         _UpperCAmelCase		:       Optional[Any]       =    self.albert(
             input_ids=A_       ,				attention_mask=A_       ,				token_type_ids=A_       ,				position_ids=A_       ,				head_mask=A_       ,				inputs_embeds=A_       ,				output_dropout=self.dropout       ,				output_layers=self.classifiers       ,				regression=self.num_labels == 1       ,				)
  
         _UpperCAmelCase		:       str       =    (logits[-1],)
  
         if labels is not None:
           _UpperCAmelCase		:       List[Any]       =    None
           _UpperCAmelCase		:       Optional[int]       =    0
           for ix, logits_item in enumerate(A_						):
             if self.num_labels == 1:
               #  We are doing regression
               _UpperCAmelCase		:       int       =    MSELoss()
               _UpperCAmelCase		:       Union[str, Any]       =    loss_fct(logits_item.view(-1						)       ,				labels.view(-1						)						)
             else:
               _UpperCAmelCase		:       List[Any]       =    CrossEntropyLoss()
               _UpperCAmelCase		:       List[Any]       =    loss_fct(logits_item.view(-1       ,				self.num_labels						)       ,				labels.view(-1						)						)
             if total_loss is None:
               _UpperCAmelCase		:       List[str]       =    loss
             else:
               total_loss += loss * (ix + 1)
             total_weights += ix + 1
           _UpperCAmelCase		:       List[str]       =    (total_loss / total_weights,) + outputs
  
         return outputs
  
  
  
  
  
  