

from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy





@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """			,			SCREAMING_SNAKE_CASE			,			)
class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):
  _a									=  RobertaConfig
  _a									=  """roberta"""




  def __init__(				self							,   lowerCAmelCase					)  -> Dict:
        '''simple docstring'''






        super().__init__(lowerCAmelCase					)

        _lowercase	       =RobertaEmbeddings(lowerCAmelCase					)
        self.init_weights()





@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """			,			SCREAMING_SNAKE_CASE			,			)
class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):
  _a									=  RobertaConfig
  _a									=  """roberta"""




  def __init__(				self							,   lowerCAmelCase					)  -> Dict:
        '''simple docstring'''






        super().__init__(lowerCAmelCase					)
        _lowercase	       =config.num_labels
        _lowercase	       =config.num_hidden_layers

        _lowercase	       =DeeRobertaModel(lowerCAmelCase					)
        _lowercase	       =nn.Dropout(config.hidden_dropout_prob					)
        _lowercase	       =nn.Linear(config.hidden_size							,   self.config.num_labels					)




  @add_start_docstrings_to_model_forward(lowerCAmelCase					)
  def   A__     (				self							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase=-1							,   lowerCAmelCase=False							,   )  -> Optional[int]:
        '''simple docstring'''






        _lowercase	       =self.num_layers
        try:
              _lowercase	       =self.roberta(
                  lowerCAmelCase							,   attention_mask=lowerCAmelCase							,   token_type_ids=lowerCAmelCase							,   position_ids=lowerCAmelCase							,   head_mask=lowerCAmelCase							,   inputs_embeds=lowerCAmelCase							,   )

              _lowercase	       =outputs[1]

              _lowercase	       =self.dropout(lowerCAmelCase					)
              _lowercase	       =self.classifier(lowerCAmelCase					)
              _lowercase	       =(logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
              _lowercase	       =e.message
              _lowercase	       =e.exit_layer
              _lowercase	       =outputs[0]

        if not self.training:
              _lowercase	       =entropy(lowerCAmelCase					)
              _lowercase	       =[]
              _lowercase	       =[]
        if labels is not None:
              if self.num_labels == 1:
                    #  We are doing regression
                    _lowercase	       =MSELoss()
                    _lowercase	       =loss_fct(logits.view(-1					)							,   labels.view(-1					)					)
              else:
                    _lowercase	       =CrossEntropyLoss()
                    _lowercase	       =loss_fct(logits.view(-1							,   self.num_labels					)							,   labels.view(-1					)					)

              # work with highway exits
              _lowercase	       =[]
              for highway_exit in outputs[-1]:
                    _lowercase	       =highway_exit[0]
                    if not self.training:
                          highway_logits_all.append(lowerCAmelCase					)
                          highway_entropy.append(highway_exit[2]					)
                    if self.num_labels == 1:
                          #  We are doing regression
                          _lowercase	       =MSELoss()
                          _lowercase	       =loss_fct(highway_logits.view(-1					)							,   labels.view(-1					)					)
                    else:
                          _lowercase	       =CrossEntropyLoss()
                          _lowercase	       =loss_fct(highway_logits.view(-1							,   self.num_labels					)							,   labels.view(-1					)					)
                    highway_losses.append(lowerCAmelCase					)

              if train_highway:
                    _lowercase	       =(sum(highway_losses[:-1]					),) + outputs
                    # exclude the final highway, of course
              else:
                    _lowercase	       =(loss,) + outputs
        if not self.training:
              _lowercase	       =outputs + ((original_entropy, highway_entropy), exit_layer)
              if output_layer >= 0:
                    _lowercase	       =(
                        (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                    )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowercase_									=   {
    'configuration_bloom': ['BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BloomConfig', 'BloomOnnxConfig'],
}
try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   ['BloomTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   [
      'BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'BloomForCausalLM',
      'BloomModel',
      'BloomPreTrainedModel',
      'BloomForSequenceClassification',
      'BloomForTokenClassification',
      'BloomForQuestionAnswering',
  ]

if TYPE_CHECKING:
  from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_bloom_fast import BloomTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_bloom import (
        BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
        BloomForCausalLM,
        BloomForQuestionAnswering,
        BloomForSequenceClassification,
        BloomForTokenClassification,
        BloomModel,
        BloomPreTrainedModel,
    )

else:
  import sys

  lowercase_									=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


