code: 
  
  
  
  """simple docstring"""
  import collections
  import inspect
  import unittest
  
  from transformers import DonutSwinConfig
  from transformers.testing_utils import require_torch, slow, torch_device
  from transformers.utils import is_torch_available
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
    import torch
    from torch import nn
  
    from transformers import DonutSwinModel
    from transformers.models.donut.modeling_donut_swin import DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST
  class 							_UpperCAmelCase							:
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A ,			A=1_3 ,			A=3_2 ,			A=2 ,			A=3 ,			A=1_6 ,			A=[1, 2, 1] ,			A=[2, 2, 4] ,			A=2 ,			A=2.0 ,			A=True ,			A=0.0 ,			A=0.0 ,			A=0.1 ,			A="gelu" ,			A=False ,			A=True ,			A=0.02 ,			A=1E-5 ,			A=True ,			A=None ,			A=True ,			A=1_0 ,			A=8 ,			)							->      Tuple:
           _UpperCAmelCase							:						Optional[int]       							=					parent
           _UpperCAmelCase							:						Union[str, Any]       							=					batch_size
           _UpperCAmelCase							:						Any       							=					image_size
           _UpperCAmelCase							:						int       							=					patch_size
           _UpperCAmelCase							:						str       							=					num_channels
           _UpperCAmelCase							:						str       							=					embed_dim
           _UpperCAmelCase							:						List[str]       							=					depths
           _UpperCAmelCase							:						str       							=					num_heads
           _UpperCAmelCase							:						Tuple       							=					window_size
           _UpperCAmelCase							:						Tuple       							=					mlp_ratio
           _UpperCAmelCase							:						Tuple       							=					qkv_bias
           _UpperCAmelCase							:						Optional[int]       							=					hidden_dropout_prob
           _UpperCAmelCase							:						Tuple       							=					attention_probs_dropout_prob
           _UpperCAmelCase							:						Optional[Any]       							=					drop_path_rate
           _UpperCAmelCase							:						Optional[Any]       							=					hidden_act
           _UpperCAmelCase							:						Optional[int]       							=					use_absolute_embeddings
           _UpperCAmelCase							:						List[str]       							=					patch_norm
           _UpperCAmelCase							:						int       							=					layer_norm_eps
           _UpperCAmelCase							:						str       							=					initializer_range
           _UpperCAmelCase							:						Union[str, Any]       							=					is_training
           _UpperCAmelCase							:						str       							=					scope
           _UpperCAmelCase							:						Dict       							=					use_labels
           _UpperCAmelCase							:						Tuple       							=					type_sequence_label_size
           _UpperCAmelCase							:						Union[str, Any]       							=					encoder_stride
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      int:
           _UpperCAmelCase							:						Any       							=					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)
  
           _UpperCAmelCase							:						int       							=					None
           if self.use_labels:
               _UpperCAmelCase							:						str       							=					ids_tensor([self.batch_size] ,			self.type_sequence_label_size	)
  
           _UpperCAmelCase							:						Optional[int]       							=					self.get_config()
  
           return config, pixel_values, labels
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      List[str]:
           return DonutSwinConfig(
               image_size=self.image_size ,			patch_size=self.patch_size ,			num_channels=self.num_channels ,			embed_dim=self.embed_dim ,			depths=self.depths ,			num_heads=self.num_heads ,			window_size=self.window_size ,			mlp_ratio=self.mlp_ratio ,			qkv_bias=self.qkv_bias ,			hidden_dropout_prob=self.hidden_dropout_prob ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob ,			drop_path_rate=self.drop_path_rate ,			hidden_act=self.hidden_act ,			use_absolute_embeddings=self.use_absolute_embeddings ,			path_norm=self.patch_norm ,			layer_norm_eps=self.layer_norm_eps ,			initializer_range=self.initializer_range ,			encoder_stride=self.encoder_stride ,			)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A ,			A	)							->      str:
           _UpperCAmelCase							:						Optional[int]       							=					DonutSwinModel(config=A	)
           model.to(A	)
           model.eval()
           _UpperCAmelCase							:						Optional[int]       							=					model(A	)
  
           _UpperCAmelCase							:						Union[str, Any]       							=					((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths	) - 1))
           _UpperCAmelCase							:						Tuple       							=					int(config.embed_dim * 2 ** (len(config.depths	) - 1)	)
  
           self.parent.assertEqual(result.last_hidden_state.shape ,			(self.batch_size, expected_seq_len, expected_dim)	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Optional[Any]:
           _UpperCAmelCase							:						Dict       							=					self.prepare_config_and_inputs()
           (
               (
               _UpperCAmelCase
           )							,       (
               _UpperCAmelCase
           )							,       (
               _UpperCAmelCase
           )							,       
           )							:						Union[str, Any]       							=					config_and_inputs
           _UpperCAmelCase							:						Dict       							=					{'''pixel_values''': pixel_values}
           return config, inputs_dict
  
  
  
  @require_torch
  class 							_UpperCAmelCase							(				a	,a	,unittest.TestCase   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       a__             =(DonutSwinModel,) if is_torch_available() else ()
       a__             ={'''feature-extraction''': DonutSwinModel} if is_torch_available() else {}
       a__             =True
  
       a__             =False
       a__             =False
       a__             =False
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
           _UpperCAmelCase							:						Union[str, Any]       							=					DonutSwinModelTester(self	)
           _UpperCAmelCase							:						Tuple       							=					ConfigTester(self ,			config_class=A ,			embed_dim=3_7	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Any:
           self.create_and_test_config_common_properties()
           self.config_tester.create_and_test_config_to_json_string()
           self.config_tester.create_and_test_config_to_json_file()
           self.config_tester.create_and_test_config_from_and_save_pretrained()
           self.config_tester.create_and_test_config_with_num_labels()
           self.config_tester.check_config_can_be_init_without_params()
           self.config_tester.check_config_arguments_init()
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      str:
           return
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Any:
           _UpperCAmelCase							:						List[str]       							=					self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*A	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Tuple:
           # DonutSwin does not use inputs_embeds
           pass
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Optional[Any]:
           _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[Any]       							=					self.model_tester.prepare_config_and_inputs_for_common()
  
           for model_class in self.all_model_classes:
               _UpperCAmelCase							:						Dict       							=					model_class(A	)
               self.assertIsInstance(model.get_input_embeddings() ,			(nn.Module)	)
               _UpperCAmelCase							:						str       							=					model.get_output_embeddings()
               self.assertTrue(x is None or isinstance(A ,			nn.Linear	)	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
           _UpperCAmelCase							,       _UpperCAmelCase							:						List[Any]       							=					self.model_tester.prepare_config_and_inputs_for_common()
  
           for model_class in self.all_model_classes:
               _UpperCAmelCase							:						Any       							=					model_class(A	)
               _UpperCAmelCase							:						Tuple       							=					inspect.signature(model.forward	)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               _UpperCAmelCase							:						List[str]       							=					[*signature.parameters.keys()]
  
               _UpperCAmelCase							:						List[Any]       							=					['''pixel_values''']
               self.assertListEqual(arg_names[:1] ,			A	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      str:
           _UpperCAmelCase							,       _UpperCAmelCase							:						Dict       							=					self.model_tester.prepare_config_and_inputs_for_common()
           _UpperCAmelCase							:						List[Any]       							=					True
  
           for model_class in self.all_model_classes:
               _UpperCAmelCase							:						Optional[Any]       							=					True
               _UpperCAmelCase							:						int       							=					False
               _UpperCAmelCase							:						List[Any]       							=					True
               _UpperCAmelCase							:						str       							=					model_class(A	)
               model.to(A	)
               model.eval()
               with torch.no_grad():
                   _UpperCAmelCase							:						List[str]       							=					model(**self._prepare_for_class(A ,			A	)	)
               _UpperCAmelCase							:						Optional[int]       							=					outputs.attentions
               _UpperCAmelCase							:						Optional[int]       							=					len(self.model_tester.depths	)
               self.assertEqual(len(A	) ,			A	)
  
               # check that output_attentions also work using config
               del inputs_dict["output_attentions"]
               _UpperCAmelCase							:						Optional[Any]       							=					True
               _UpperCAmelCase							:						Any       							=					config.window_size**2
               _UpperCAmelCase							:						Optional[int]       							=					model_class(A	)
               model.to(A	)
               model.eval()
               with torch.no_grad():
                   _UpperCAmelCase							:						List[str]       							=					model(**self._prepare_for_class(A ,			A	)	)
               _UpperCAmelCase							:						Dict       							=					outputs.attentions
               self.assertEqual(len(A	) ,			A	)
  
               self.assertListEqual(
                   list(attentions[0].shape[-3:]	) ,			[self.model_tester.num_heads[0], window_size_squared, window_size_squared] ,			)
               _UpperCAmelCase							:						int       							=					len(A	)
  
               # Check attention is always last and order is fine
               _UpperCAmelCase							:						Optional[int]       							=					True
               _UpperCAmelCase							:						Tuple       							=					True
               _UpperCAmelCase							:						Tuple       							=					model_class(A	)
               model.to(A	)
               model.eval()
               with torch.no_grad():
                   _UpperCAmelCase							:						Tuple       							=					model(**self._prepare_for_class(A ,			A	)	)
  
               if hasattr(self.model_tester ,			'''num_hidden_states_types'''	):
                   _UpperCAmelCase							:						int       							=					self.model_tester.num_hidden_states_types
               else:
                   # also another +1 for reshaped_hidden_states
                   _UpperCAmelCase							:						int       							=					2
               self.assertEqual(out_len + added_hidden_states ,			len(A	)	)
  
               _UpperCAmelCase							:						Optional[Any]       							=					outputs.attentions
  
               self.assertEqual(len(A	) ,			A	)
  
               self.assertListEqual(
                   list(self_attentions[0].shape[-3:]	) ,			[self.model_tester.num_heads[0], window_size_squared, window_size_squared] ,			)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A ,			A ,			A	)							->      Dict:
           _UpperCAmelCase							:						Tuple       							=					model_class(A	)
           model.to(A	)
           model.eval()
  
           with torch.no_grad():
               _UpperCAmelCase							:						List[str]       							=					model(**self._prepare_for_class(A ,			A	)	)
  
           _UpperCAmelCase							:						Optional[int]       							=					outputs.hidden_states
  
           _UpperCAmelCase							:						List[str]       							=					getattr(
               self.model_tester ,			'''expected_num_hidden_layers''' ,			len(self.model_tester.depths	) + 1	)
           self.assertEqual(len(A	) ,			A	)
  
           # DonutSwin has a different seq_length
           _UpperCAmelCase							:						Optional[int]       							=					(
               config.patch_size
               if isinstance(config.patch_size ,			collections.abc.Iterable	)
               else (config.patch_size, config.patch_size)
           )
  
           _UpperCAmelCase							:						Any       							=					(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
  
           self.assertListEqual(
               list(hidden_states[0].shape[-2:]	) ,			[num_patches, self.model_tester.embed_dim] ,			)
  
           _UpperCAmelCase							:						int       							=					outputs.reshaped_hidden_states
           self.assertEqual(len(A	) ,			A	)
  
           _UpperCAmelCase							,       _UpperCAmelCase							,       _UpperCAmelCase							,       _UpperCAmelCase							:						Union[str, Any]       							=					reshaped_hidden_states[0].shape
           _UpperCAmelCase							:						Dict       							=					(
               reshaped_hidden_states[0].view(A ,			A ,			height * width	).permute(0 ,			2 ,			1	)
           )
           self.assertListEqual(
               list(reshaped_hidden_states.shape[-2:]	) ,			[num_patches, self.model_tester.embed_dim] ,			)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Optional[int]:
           _UpperCAmelCase							,       _UpperCAmelCase							:						str       							=					self.model_tester.prepare_config_and_inputs_for_common()
  
           _UpperCAmelCase							:						str       							=					(
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size ,			collections.abc.Iterable	)
               else (self.model_tester.image_size, self.model_tester.image_size)
           )
  
           for model_class in self.all_model_classes:
               _UpperCAmelCase							:						List[Any]       							=					True
               self.check_hidden_states_output(A ,			A ,			A ,			A	)
  
               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               _UpperCAmelCase							:						Optional[Any]       							=					True
  
               self.check_hidden_states_output(A ,			A ,			A ,			A	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      List[Any]:
           _UpperCAmelCase							,       _UpperCAmelCase							:						Dict       							=					self.model_tester.prepare_config_and_inputs_for_common()
           _UpperCAmelCase							:						str       							=					3
  
           _UpperCAmelCase							:						Dict       							=					(
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size ,			collections.abc.Iterable	)
               else (self.model_tester.image_size, self.model_tester.image_size)
           )
           _UpperCAmelCase							:						str       							=					(
               config.patch_size
               if isinstance(config.patch_size ,			collections.abc.Iterable	)
               else (config.patch_size, config.patch_size)
           )
  
           _UpperCAmelCase							:						int       							=					image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
           _UpperCAmelCase							:						int       							=					image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])
  
           for model_class in self.all_model_classes:
               _UpperCAmelCase							:						Any       							=					True
               self.check_hidden_states_output(A ,			A ,			A ,			(padded_height, padded_width)	)
  
               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               _UpperCAmelCase							:						str       							=					True
               self.check_hidden_states_output(A ,			A ,			A ,			(padded_height, padded_width)	)
  
  
  
  
  
       @slow
       def 				__lowerCAmelCase			(      self	)							->      int:
           for model_name in DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _UpperCAmelCase							:						Any       							=					DonutSwinModel.from_pretrained(A	)
               self.assertIsNotNone(A	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Any:
           _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					self.model_tester.prepare_config_and_inputs_for_common()
  
           _UpperCAmelCase							:						str       							=					_config_zero_init(A	)
           for model_class in self.all_model_classes:
               _UpperCAmelCase							:						Optional[Any]       							=					model_class(config=A	)
               for name, param in model.named_parameters():
                   if "embeddings" not in name and param.requires_grad:
                       self.assertIn(
                           ((param.data.mean() * 1E9).round() / 1E9).item() ,			[0.0, 1.0] ,			msg=f'Parameter {name} of model {model_class} seems not properly initialized' ,			)
  
  
  
style_context: 
  
  
  
  """simple docstring"""
  import logging
  
  import torch
  from torch import nn
  from torch.nn import CrossEntropyLoss, MSELoss
  
  from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
  from transformers.models.albert.modeling_albert import (
      ALBERT_INPUTS_DOCSTRING,
      ALBERT_START_DOCSTRING,
      AlbertModel,
      AlbertPreTrainedModel,
      AlbertTransformer,
  )
  
  
  _lowerCAmelCase					:Union[str, Any]  =      logging.getLogger(__name__)
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A ,			A=None ,			A=None	)							->      Tuple:
           if current_layer == 0:
               _UpperCAmelCase							:						Optional[Any]       							=					self.embedding_hidden_mapping_in(A	)
           else:
               _UpperCAmelCase							:						Union[str, Any]       							=					hidden_states[0]
  
           _UpperCAmelCase							:						Dict       							=					int(self.config.num_hidden_layers / self.config.num_hidden_groups	)
  
           # Index of the hidden group
           _UpperCAmelCase							:						Dict       							=					int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)	)
  
           _UpperCAmelCase							:						List[Any]       							=					self.albert_layer_groups[group_idx](
               A ,			A ,			head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group] ,			)
           _UpperCAmelCase							:						str       							=					layer_group_output[0]
  
           return (hidden_states,)
  
  
  
  @add_start_docstrings(
      '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''	,a	,)
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A	)							->      str:
           super().__init__(A	)
  
           _UpperCAmelCase							:						Dict       							=					AlbertTransformerWithPabee(A	)
  
           self.init_weights()
           _UpperCAmelCase							:						str       							=					0
           _UpperCAmelCase							:						Union[str, Any]       							=					0
           _UpperCAmelCase							:						Dict       							=					0
  
           _UpperCAmelCase							:						str       							=					0
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A	)							->      Optional[int]:
           _UpperCAmelCase							:						List[Any]       							=					threshold
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A	)							->      Optional[Any]:
           _UpperCAmelCase							:						int       							=					patience
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      Any:
           _UpperCAmelCase							:						Dict       							=					0
           _UpperCAmelCase							:						List[str]       							=					0
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      List[Any]:
           _UpperCAmelCase							:						Union[str, Any]       							=					self.inference_layers_num / self.inference_instances_num
           _UpperCAmelCase							:						str       							=					(
               f'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
               f' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
           )
           print(A	)
  
  
  
  
  
       @add_start_docstrings_to_model_forward(A	)
       def 				__lowerCAmelCase			(      self ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=False ,			)							->      int:
  
           if input_ids is not None and inputs_embeds is not None:
               raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''	)
           elif input_ids is not None:
               _UpperCAmelCase							:						Optional[Any]       							=					input_ids.size()
           elif inputs_embeds is not None:
               _UpperCAmelCase							:						Optional[Any]       							=					inputs_embeds.size()[:-1]
           else:
               raise ValueError('''You have to specify either input_ids or inputs_embeds'''	)
  
           _UpperCAmelCase							:						Tuple       							=					input_ids.device if input_ids is not None else inputs_embeds.device
  
           if attention_mask is None:
               _UpperCAmelCase							:						Union[str, Any]       							=					torch.ones(A ,			device=A	)
           if token_type_ids is None:
               _UpperCAmelCase							:						Tuple       							=					torch.zeros(A ,			dtype=torch.long ,			device=A	)
  
           _UpperCAmelCase							:						Optional[Any]       							=					attention_mask.unsqueeze(1	).unsqueeze(2	)
           _UpperCAmelCase							:						Union[str, Any]       							=					extended_attention_mask.to(dtype=self.dtype	)  # fp16 compatibility
           _UpperCAmelCase							:						Dict       							=					(1.0 - extended_attention_mask) * -10_000.0
           _UpperCAmelCase							:						Any       							=					self.get_head_mask(A ,			self.config.num_hidden_layers	)
  
           _UpperCAmelCase							:						int       							=					self.embeddings(
               A ,			position_ids=A ,			token_type_ids=A ,			inputs_embeds=A	)
           _UpperCAmelCase							:						Any       							=					embedding_output
  
           if self.training:
               _UpperCAmelCase							:						Optional[Any]       							=					[]
               for i in range(self.config.num_hidden_layers	):
                   _UpperCAmelCase							:						Union[str, Any]       							=					self.encoder.adaptive_forward(
                       A ,			current_layer=A ,			attention_mask=A ,			head_mask=A ,			)
  
                   _UpperCAmelCase							:						List[str]       							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]	)	)
                   _UpperCAmelCase							:						Dict       							=					output_layers[i](output_dropout(A	)	)
                   res.append(A	)
           elif self.patience == 0:  # Use all layers for inference
               _UpperCAmelCase							:						Dict       							=					self.encoder(A ,			A ,			head_mask=A	)
               _UpperCAmelCase							:						Tuple       							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]	)	)
               _UpperCAmelCase							:						Optional[int]       							=					[output_layers[self.config.num_hidden_layers - 1](A	)]
           else:
               _UpperCAmelCase							:						List[Any]       							=					0
               _UpperCAmelCase							:						int       							=					None
               _UpperCAmelCase							:						int       							=					0
               for i in range(self.config.num_hidden_layers	):
                   calculated_layer_num += 1
                   _UpperCAmelCase							:						Tuple       							=					self.encoder.adaptive_forward(
                       A ,			current_layer=A ,			attention_mask=A ,			head_mask=A ,			)
  
                   _UpperCAmelCase							:						Any       							=					self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]	)	)
                   _UpperCAmelCase							:						Optional[Any]       							=					output_layers[i](A	)
                   if regression:
                       _UpperCAmelCase							:						str       							=					logits.detach()
                       if patient_result is not None:
                           _UpperCAmelCase							:						str       							=					patient_result.detach()
                       if (patient_result is not None) and torch.abs(patient_result - labels	) < self.regression_threshold:
                           patient_counter += 1
                       else:
                           _UpperCAmelCase							:						List[Any]       							=					0
                   else:
                       _UpperCAmelCase							:						List[str]       							=					logits.detach().argmax(dim=1	)
                       if patient_result is not None:
                           _UpperCAmelCase							:						Optional[Any]       							=					patient_result.detach().argmax(dim=1	)
                       if (patient_result is not None) and torch.all(labels.eq(A	)	):
                           patient_counter += 1
                       else:
                           _UpperCAmelCase							:						Tuple       							=					0
  
                   _UpperCAmelCase							:						Any       							=					logits
                   if patient_counter == self.patience:
                       break
               _UpperCAmelCase							:						List[str]       							=					[patient_result]
               self.inference_layers_num += calculated_layer_num
               self.inference_instances_num += 1
  
           return res
  
  
  
  @add_start_docstrings(
      '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
      the pooled output) e.g. for GLUE tasks. '''	,a	,)
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A	)							->      Optional[int]:
           super().__init__(A	)
           _UpperCAmelCase							:						List[Any]       							=					config.num_labels
  
           _UpperCAmelCase							:						Optional[int]       							=					AlbertModelWithPabee(A	)
           _UpperCAmelCase							:						List[str]       							=					nn.Dropout(config.classifier_dropout_prob	)
           _UpperCAmelCase							:						Optional[Any]       							=					nn.ModuleList(
               [nn.Linear(config.hidden_size ,			self.config.num_labels	) for _ in range(config.num_hidden_layers	)]	)
  
           self.init_weights()
  
  
  
  
  
       @add_start_docstrings_to_model_forward(A	)
       def 				__lowerCAmelCase			(      self ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			)							->      List[str]:
           _UpperCAmelCase							:						Optional[Any]       							=					self.albert(
               input_ids=A ,			attention_mask=A ,			token_type_ids=A ,			position_ids=A ,			head_mask=A ,			inputs_embeds=A ,			output_dropout=self.dropout ,			output_layers=self.classifiers ,			regression=self.num_labels == 1 ,			)
  
           _UpperCAmelCase							:						int       							=					(logits[-1],)
  
           if labels is not None:
               _UpperCAmelCase							:						int       							=					None
               _UpperCAmelCase							:						Tuple       							=					0
               for ix, logits_item in enumerate(A	):
                   if self.num_labels == 1:
                       #  We are doing regression
                       _UpperCAmelCase							:						Dict       							=					MSELoss()
                       _UpperCAmelCase							:						Optional[int]       							=					loss_fct(logits_item.view(-1	) ,			labels.view(-1	)	)
                   else:
                       _UpperCAmelCase							:						List[str]       							=					CrossEntropyLoss()
                       _UpperCAmelCase							:						Any       							=					loss_fct(logits_item.view(-1 ,			self.num_labels	) ,			labels.view(-1	)	)
                   if total_loss is None:
                       _UpperCAmelCase							:						Optional[Any]       							=					loss
                   else:
                       total_loss += loss * (ix + 1)
                   total_weights += ix + 1
               _UpperCAmelCase							:						Any       							=					(total_loss / total_weights,) + outputs
  
           return outputs
  
  
  