
import inspect
import unittest
from typing import List

import numpy as np

from transformers import EfficientFormerConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFEfficientFormerForImageClassification,
				    TFEfficientFormerForImageClassificationWithTeacher,
				    TFEfficientFormerModel,
				)
				from transformers.models.efficientformer.modeling_tf_efficientformer import (
				    TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				)


if is_vision_available():
				from PIL import Image

				from transformers import EfficientFormerImageProcessor




class 						lowercase__  :
	def __init__(			self							:   Any       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int = 13       ,   UpperCAmelCase_							:   int = 64       ,   UpperCAmelCase_							:   int = 2       ,   UpperCAmelCase_							:   int = 3       ,   UpperCAmelCase_							:   int = 3       ,   UpperCAmelCase_							:   bool = True       ,   UpperCAmelCase_							:   bool = True       ,   UpperCAmelCase_							:   int = 128       ,   UpperCAmelCase_							:   str=[16, 32, 64, 128]       ,   UpperCAmelCase_							:   int = 7       ,   UpperCAmelCase_							:   int = 4       ,   UpperCAmelCase_							:   int = 37       ,   UpperCAmelCase_							:   str = "gelu"       ,   UpperCAmelCase_							:   float = 0.1       ,   UpperCAmelCase_							:   float = 0.1       ,   UpperCAmelCase_							:   int = 10       ,   UpperCAmelCase_							:   float = 0.02       ,   UpperCAmelCase_							:   int = 2       ,   UpperCAmelCase_							:   int = 1       ,   UpperCAmelCase_							:   int = 128       ,   UpperCAmelCase_							:   List[int] = [2, 2, 2, 2]       ,   UpperCAmelCase_							:   int = 2       ,   UpperCAmelCase_							:   int = 2       ,   ):
						SCREAMING_SNAKE_CASE__													= parent
						SCREAMING_SNAKE_CASE__													= batch_size
						SCREAMING_SNAKE_CASE__													= image_size
						SCREAMING_SNAKE_CASE__													= patch_size
						SCREAMING_SNAKE_CASE__													= num_channels
						SCREAMING_SNAKE_CASE__													= is_training
						SCREAMING_SNAKE_CASE__													= use_labels
						SCREAMING_SNAKE_CASE__													= hidden_size
						SCREAMING_SNAKE_CASE__													= num_hidden_layers
						SCREAMING_SNAKE_CASE__													= num_attention_heads
						SCREAMING_SNAKE_CASE__													= intermediate_size
						SCREAMING_SNAKE_CASE__													= hidden_act
						SCREAMING_SNAKE_CASE__													= hidden_dropout_prob
						SCREAMING_SNAKE_CASE__													= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE__													= type_sequence_label_size
						SCREAMING_SNAKE_CASE__													= initializer_range
						SCREAMING_SNAKE_CASE__													= encoder_stride
						SCREAMING_SNAKE_CASE__													= num_attention_outputs
						SCREAMING_SNAKE_CASE__													= embed_dim
						SCREAMING_SNAKE_CASE__													= embed_dim + 1
						SCREAMING_SNAKE_CASE__													= resolution
						SCREAMING_SNAKE_CASE__													= depths
						SCREAMING_SNAKE_CASE__													= hidden_sizes
						SCREAMING_SNAKE_CASE__													= dim
						SCREAMING_SNAKE_CASE__													= mlp_expansion_ratio
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

						SCREAMING_SNAKE_CASE__													= None
						if self.use_labels:
											SCREAMING_SNAKE_CASE__													= ids_tensor([self.batch_size]       ,   self.type_sequence_label_size							)

						SCREAMING_SNAKE_CASE__													= self.get_config()

						return config, pixel_values, labels
	def      A_				(			self							:   Optional[Any]							):
						return EfficientFormerConfig(
						    image_size=self.image_size       ,   patch_size=self.patch_size       ,   num_channels=self.num_channels       ,   hidden_size=self.hidden_size       ,   num_hidden_layers=self.num_hidden_layers       ,   num_attention_heads=self.num_attention_heads       ,   intermediate_size=self.intermediate_size       ,   hidden_act=self.hidden_act       ,   hidden_dropout_prob=self.hidden_dropout_prob       ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,   is_decoder=UpperCAmelCase_       ,   initializer_range=self.initializer_range       ,   encoder_stride=self.encoder_stride       ,   resolution=self.resolution       ,   depths=self.depths       ,   hidden_sizes=self.hidden_sizes       ,   dim=self.dim       ,   mlp_expansion_ratio=self.mlp_expansion_ratio       ,   )
	def      A_				(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   List[str]       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= TFEfficientFormerModel(config=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= model(UpperCAmelCase_       ,   training=UpperCAmelCase_							)
						self.parent.assertEqual(result.last_hidden_state.shape       ,   (self.batch_size, self.seq_length, self.hidden_size)							)
	def      A_				(			self							:   Dict       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   Tuple       ,   UpperCAmelCase_							:   Optional[Any]							):
						SCREAMING_SNAKE_CASE__													= self.type_sequence_label_size
						SCREAMING_SNAKE_CASE__													= TFEfficientFormerForImageClassification(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= model(UpperCAmelCase_       ,   labels=UpperCAmelCase_       ,   training=UpperCAmelCase_							)
						self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.type_sequence_label_size)							)

						# test greyscale images
						SCREAMING_SNAKE_CASE__													= 1
						SCREAMING_SNAKE_CASE__													= TFEfficientFormerForImageClassification(UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
						SCREAMING_SNAKE_CASE__													= model(UpperCAmelCase_       ,   labels=UpperCAmelCase_							)
						self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.type_sequence_label_size)							)





	def      A_				(			self							:   Dict							):
						SCREAMING_SNAKE_CASE__													= self.prepare_config_and_inputs()
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= config_and_inputs
						SCREAMING_SNAKE_CASE__													= {'pixel_values': pixel_values}
						return config, inputs_dict





@require_tf
class 						lowercase__  (						_UpperCAmelCase     ,       _UpperCAmelCase     ,       unittest.TestCase  ):
	A__    :   Union[str, Any]				       =(
	    (
	        TFEfficientFormerModel,
	        TFEfficientFormerForImageClassificationWithTeacher,
	        TFEfficientFormerForImageClassification,
	    )
	    if is_tf_available()
	    else ()
	)
	A__    :   Any				       =(
	    {
	        """feature-extraction""": TFEfficientFormerModel,
	        """image-classification""": (
	            TFEfficientFormerForImageClassification,
	            TFEfficientFormerForImageClassificationWithTeacher,
	        ),
	    }
	    if is_tf_available()
	    else {}
	)

	A__    :   Tuple				       =False

	A__    :   Union[str, Any]				       =False
	A__    :   Optional[int]				       =False
	A__    :   Optional[Any]				       =False
	A__    :   Dict				       =False
	def      A_				(			self							:   int							):
						SCREAMING_SNAKE_CASE__													= TFEfficientFormerModelTester(self							)
						SCREAMING_SNAKE_CASE__													= ConfigTester(
						    self       ,   config_class=UpperCAmelCase_       ,   has_text_modality=UpperCAmelCase_       ,   hidden_size=37							)
	def      A_				(			self							:   str							):
						self.config_tester.run_common_tests()
	@unittest.skip(reason='EfficientFormer does not use inputs_embeds'							)
	def      A_				(			self							:   Union[str, Any]							):
						pass
	@unittest.skip(reason='EfficientFormer does not support input and output embeddings'							)
	def      A_				(			self							:   List[str]							):
						pass
	def      A_				(			self							:   Dict							):
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											SCREAMING_SNAKE_CASE__													= model_class(UpperCAmelCase_							)
											SCREAMING_SNAKE_CASE__													= inspect.signature(model.call							)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											SCREAMING_SNAKE_CASE__													= [*signature.parameters.keys()]

											SCREAMING_SNAKE_CASE__													= ['pixel_values']
											self.assertListEqual(arg_names[:1]       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Optional[Any]							):
						def check_hidden_states_output(UpperCAmelCase_							:   Any       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   Union[str, Any]							):
											SCREAMING_SNAKE_CASE__													= model_class(UpperCAmelCase_							)

											SCREAMING_SNAKE_CASE__													= model(**self._prepare_for_class(UpperCAmelCase_       ,   UpperCAmelCase_							)       ,   training=UpperCAmelCase_							)
											SCREAMING_SNAKE_CASE__													= outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

											SCREAMING_SNAKE_CASE__													= getattr(
											    self.model_tester       ,   'expected_num_hidden_layers'       ,   self.model_tester.num_hidden_layers + 1							)
											self.assertEqual(len(UpperCAmelCase_							)       ,   UpperCAmelCase_							)

											if hasattr(self.model_tester       ,   'encoder_seq_length'							):
																SCREAMING_SNAKE_CASE__													= self.model_tester.encoder_seq_length
																if hasattr(self.model_tester       ,   'chunk_length'							) and self.model_tester.chunk_length > 1:
																					SCREAMING_SNAKE_CASE__													= seq_length * self.model_tester.chunk_length
											else:
																SCREAMING_SNAKE_CASE__													= self.model_tester.seq_length

											self.assertListEqual(
											    list(hidden_states[-1].shape[-2:]							)       ,   [seq_length, self.model_tester.hidden_size]       ,   )

											if config.is_encoder_decoder:
																SCREAMING_SNAKE_CASE__													= outputs.decoder_hidden_states

																self.asseretIsInstance(UpperCAmelCase_       ,   (list, tuple)							)
																self.assertEqual(len(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
																SCREAMING_SNAKE_CASE__													= getattr(self.model_tester       ,   'seq_length'       ,   UpperCAmelCase_							)
																SCREAMING_SNAKE_CASE__													= getattr(self.model_tester       ,   'decoder_seq_length'       ,   UpperCAmelCase_							)

																self.assertListEqual(
																    list(hidden_states[-1].shape[-2:]							)       ,   [decoder_seq_length, self.model_tester.hidden_size]       ,   )

						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											SCREAMING_SNAKE_CASE__													= True
											check_hidden_states_output(UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											SCREAMING_SNAKE_CASE__													= True

											check_hidden_states_output(UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Dict       ,   UpperCAmelCase_							:   List[Any]       ,   UpperCAmelCase_							:   Any       ,   UpperCAmelCase_							:   Union[str, Any]=False							):
						SCREAMING_SNAKE_CASE__													= super()._prepare_for_class(UpperCAmelCase_       ,   UpperCAmelCase_       ,   return_labels=UpperCAmelCase_							)

						if return_labels:
											if model_class.__name__ == "TFEfficientFormerForImageClassificationWithTeacher":
																del inputs_dict["labels"]

						return inputs_dict
	def      A_				(			self							:   Dict							):
						SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCAmelCase_							)
	@unittest.skip(reason='EfficientFormer does not implement masked image modeling yet'							)
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_image_modeling(*UpperCAmelCase_							)
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*UpperCAmelCase_							)
	@slow
	def      A_				(			self							:   Union[str, Any]							):
						for model_name in TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											SCREAMING_SNAKE_CASE__													= TFEfficientFormerModel.from_pretrained(UpperCAmelCase_							)
											self.assertIsNotNone(UpperCAmelCase_							)
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs_for_common()
						SCREAMING_SNAKE_CASE__													= True

						SCREAMING_SNAKE_CASE__													= getattr(self.model_tester       ,   'seq_length'       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= getattr(self.model_tester       ,   'encoder_seq_length'       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= getattr(self.model_tester       ,   'key_length'       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= getattr(self.model_tester       ,   'chunk_length'       ,   UpperCAmelCase_							)

						if chunk_length is not None and hasattr(self.model_tester       ,   'num_hashes'							):
											SCREAMING_SNAKE_CASE__													= encoder_seq_length * self.model_tester.num_hashes

						for model_class in self.all_model_classes:
											SCREAMING_SNAKE_CASE__													= True
											SCREAMING_SNAKE_CASE__													= False
											SCREAMING_SNAKE_CASE__													= True
											SCREAMING_SNAKE_CASE__													= model_class(UpperCAmelCase_							)

											SCREAMING_SNAKE_CASE__													= model(**self._prepare_for_class(UpperCAmelCase_       ,   UpperCAmelCase_							)       ,   training=UpperCAmelCase_							)
											SCREAMING_SNAKE_CASE__													= outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
											self.assertEqual(len(UpperCAmelCase_							)       ,   self.model_tester.num_attention_outputs							)

											# check that output_attentions also work using config
											del inputs_dict["output_attentions"]
											SCREAMING_SNAKE_CASE__													= True
											SCREAMING_SNAKE_CASE__													= model_class(UpperCAmelCase_							)
											SCREAMING_SNAKE_CASE__													= model(**self._prepare_for_class(UpperCAmelCase_       ,   UpperCAmelCase_							)       ,   training=UpperCAmelCase_							)

											SCREAMING_SNAKE_CASE__													= outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
											self.assertEqual(len(UpperCAmelCase_							)       ,   self.model_tester.num_attention_outputs							)

											if chunk_length is not None:
																self.assertListEqual(
																    list(attentions[0].shape[-4:]							)       ,   [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length]       ,   )
											else:
																self.assertListEqual(
																    list(attentions[0].shape[-3:]							)       ,   [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]       ,   )





	def      A_				(			self							:   Dict							):
						# We use a simplified version of this test for EfficientFormer because it requires training=False
						# and Keras refuses to let us force that during functional construction
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											# Prepare our model
											SCREAMING_SNAKE_CASE__													= model_class(UpperCAmelCase_							)
											# These are maximally general inputs for the model, with multiple None dimensions
											# Hopefully this will catch any conditionals that fail for flexible shapes
											SCREAMING_SNAKE_CASE__													= {
											    key: tf.keras.Input(shape=val.shape[1:]       ,   dtype=val.dtype       ,   name=UpperCAmelCase_							)
											    for key, val in model.input_signature.items()
											    if key in model.dummy_inputs
											}
											SCREAMING_SNAKE_CASE__													= model(UpperCAmelCase_							)
											self.assertTrue(outputs_dict is not None							)






def 							_lowercase							(     )		->		Tuple:
					'''simple docstring'''


					SCREAMING_SNAKE_CASE__													= Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'		)
					return image




@require_tf
@require_vision
class 						lowercase__  (						unittest.TestCase  ):
	@cached_property
	def      A_				(			self							:   List[Any]							):
						return (
						    EfficientFormerImageProcessor.from_pretrained('snap-research/efficientformer-l1-300'							)
						    if is_vision_available()
						    else None
						)
	@slow
	def      A_				(			self							:   Optional[Any]							):
						SCREAMING_SNAKE_CASE__													= TFEfficientFormerForImageClassification.from_pretrained('snap-research/efficientformer-l1-300'							)
						SCREAMING_SNAKE_CASE__													= self.default_image_processor
						SCREAMING_SNAKE_CASE__													= prepare_img()
						SCREAMING_SNAKE_CASE__													= image_processor(images=UpperCAmelCase_       ,   return_tensors='tf'							)
						# forward pass
						SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   training=UpperCAmelCase_							)
						# verify the logits
						SCREAMING_SNAKE_CASE__													= tf.TensorShape((1, 1000)							)
						self.assertEqual(outputs.logits.shape       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tf.constant([-0.0_555, 0.4_825, -0.0_852]							)
						self.assertTrue(np.allclose(outputs.logits[0, :3]       ,   UpperCAmelCase_       ,   atol=1e-4							)							)





	@slow
	def      A_				(			self							:   Dict							):
						SCREAMING_SNAKE_CASE__													= TFEfficientFormerForImageClassificationWithTeacher.from_pretrained(
						    'snap-research/efficientformer-l1-300'							)
						SCREAMING_SNAKE_CASE__													= self.default_image_processor
						SCREAMING_SNAKE_CASE__													= prepare_img()
						SCREAMING_SNAKE_CASE__													= image_processor(images=UpperCAmelCase_       ,   return_tensors='tf'							)
						# forward pass
						SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   training=UpperCAmelCase_							)
						# verify the logits
						SCREAMING_SNAKE_CASE__													= tf.TensorShape((1, 1000)							)
						self.assertEqual(outputs.logits.shape       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tf.constant([-0.1_312, 0.4_353, -1.0_499]							)
						self.assertTrue(np.allclose(outputs.logits[0, :3]       ,   UpperCAmelCase_       ,   atol=1e-4							)							)







from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
				from fairscale.optim import OSS


__snake_case								=  logging.get_logger(__name__)

__snake_case								=  {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}




class 						lowercase__  (						_UpperCAmelCase  ):
	def __init__(			self							:   str       ,   UpperCAmelCase_							:   int=None       ,   UpperCAmelCase_							:   List[str]=None       ,   *UpperCAmelCase_							:   Union[str, Any]       ,   **UpperCAmelCase_							:   Optional[Any]							):
						super().__init__(*UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if config is None:
											assert isinstance(self.model       ,   UpperCAmelCase_							), (
											    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
											    F' {self.model.__class__}'
											)
											SCREAMING_SNAKE_CASE__													= self.model.config
						else:
											SCREAMING_SNAKE_CASE__													= config

						SCREAMING_SNAKE_CASE__													= data_args
						SCREAMING_SNAKE_CASE__													= self.config.tgt_vocab_size if isinstance(self.config       ,   UpperCAmelCase_							) else self.config.vocab_size

						if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
											assert self.config.pad_token_id is not None, (
											    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
											    " calculation or doing label smoothing."
											)

						if self.config.pad_token_id is None and self.config.eos_token_id is not None:
											logger.warning(
											    F'The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'
											    ' padding..'							)

						if self.args.label_smoothing == 0:
											SCREAMING_SNAKE_CASE__													= torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id							)
						else:
											# dynamically import label_smoothed_nll_loss
											from utils import label_smoothed_nll_loss

											SCREAMING_SNAKE_CASE__													= label_smoothed_nll_loss
	def      A_				(			self							:   Tuple       ,   UpperCAmelCase_							:   int							):
						if self.optimizer is None:
											SCREAMING_SNAKE_CASE__													= ['bias', 'LayerNorm.weight']
											SCREAMING_SNAKE_CASE__													= [
											    {
											        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay							)],
											        'weight_decay': self.args.weight_decay,
											    },
											    {
											        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay							)],
											        'weight_decay': 0.0,
											    },
											]
											SCREAMING_SNAKE_CASE__													= Adafactor if self.args.adafactor else AdamW
											if self.args.adafactor:
																SCREAMING_SNAKE_CASE__													= Adafactor
																SCREAMING_SNAKE_CASE__													= {'scale_parameter': False, 'relative_step': False}
											else:
																SCREAMING_SNAKE_CASE__													= AdamW
																SCREAMING_SNAKE_CASE__													= {
																    'betas': (self.args.adam_betaa, self.args.adam_betaa),
																    'eps': self.args.adam_epsilon,
																}
											SCREAMING_SNAKE_CASE__													= self.args.learning_rate
											if self.sharded_ddp:
																SCREAMING_SNAKE_CASE__													= OSS(
																    params=UpperCAmelCase_       ,   optim=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )
											else:
																SCREAMING_SNAKE_CASE__													= optimizer_cls(UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if self.lr_scheduler is None:
											SCREAMING_SNAKE_CASE__													= self._get_lr_scheduler(UpperCAmelCase_							)
						else:  # ignoring --lr_scheduler
											logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'							)
	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= arg_to_scheduler[self.args.lr_scheduler]
						if self.args.lr_scheduler == "constant":
											SCREAMING_SNAKE_CASE__													= schedule_func(self.optimizer							)
						elif self.args.lr_scheduler == "constant_w_warmup":
											SCREAMING_SNAKE_CASE__													= schedule_func(self.optimizer       ,   num_warmup_steps=self.args.warmup_steps							)
						else:
											SCREAMING_SNAKE_CASE__													= schedule_func(
											    self.optimizer       ,   num_warmup_steps=self.args.warmup_steps       ,   num_training_steps=UpperCAmelCase_							)
						return scheduler
	def      A_				(			self							:   List[str]							):
						if isinstance(self.train_dataset       ,   torch.utils.data.IterableDataset							):
											return None
						elif is_torch_tpu_available():
											return get_tpu_sampler(self.train_dataset							)
						else:
											if self.args.sortish_sampler:
																self.train_dataset.make_sortish_sampler(
																    self.args.per_device_train_batch_size       ,   distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)       ,   )

											return (
											    RandomSampler(self.train_dataset							)
											    if self.args.local_rank == -1
											    else DistributedSampler(self.train_dataset							)
											)
	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   Tuple       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Dict							):
						if self.args.label_smoothing == 0:
											if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																# force training to ignore pad token
																SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   use_cache=UpperCAmelCase_							)[0]
																SCREAMING_SNAKE_CASE__													= self.loss_fn(logits.view(-1       ,   logits.shape[-1]							)       ,   labels.view(-1							)							)
											else:
																# compute usual loss via models
																SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   labels=UpperCAmelCase_       ,   use_cache=UpperCAmelCase_							)[:2]
						else:
											# compute label smoothed loss
											SCREAMING_SNAKE_CASE__													= model(**UpperCAmelCase_       ,   use_cache=UpperCAmelCase_							)[0]
											SCREAMING_SNAKE_CASE__													= torch.nn.functional.log_softmax(UpperCAmelCase_       ,   dim=-1							)
											SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.loss_fn(UpperCAmelCase_       ,   UpperCAmelCase_       ,   self.args.label_smoothing       ,   ignore_index=self.config.pad_token_id							)
						return loss, logits
	def      A_				(			self							:   Union[str, Any]       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= inputs.pop('labels'							)
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self._compute_loss(UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
						return loss
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   nn.Module       ,   UpperCAmelCase_							:   Dict[str, Union[torch.Tensor, Any]]       ,   UpperCAmelCase_							:   bool       ,   UpperCAmelCase_							:   Optional[List[str]] = None       ,   ):
						SCREAMING_SNAKE_CASE__													= self._prepare_inputs(UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= {
						    'max_length': self.data_args.val_max_target_length
						    if self.data_args is not None
						    else self.config.max_length,
						    'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
						}

						if self.args.predict_with_generate and not self.args.prediction_loss_only:
											SCREAMING_SNAKE_CASE__													= self.model.generate(
											    inputs['input_ids']       ,   attention_mask=inputs['attention_mask']       ,   **UpperCAmelCase_       ,   )
											# in case the batch is shorter than max length, the output should be padded
											if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																SCREAMING_SNAKE_CASE__													= self._pad_tensors_to_max_len(UpperCAmelCase_       ,   gen_kwargs['max_length']							)

						SCREAMING_SNAKE_CASE__													= inputs.pop('labels'							)
						with torch.no_grad():
											# compute loss on predict data
											SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self._compute_loss(UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= loss.mean().detach()
						if self.args.prediction_loss_only:
											return (loss, None, None)

						SCREAMING_SNAKE_CASE__													= generated_tokens if self.args.predict_with_generate else logits

						if labels.shape[-1] < gen_kwargs["max_length"]:
											SCREAMING_SNAKE_CASE__													= self._pad_tensors_to_max_len(UpperCAmelCase_       ,   gen_kwargs['max_length']							)

						return (loss, logits, labels)





	def      A_				(			self							:   Union[str, Any]       ,   UpperCAmelCase_							:   Tuple       ,   UpperCAmelCase_							:   List[str]							):
						# If PAD token is not defined at least EOS token has to be defined
						SCREAMING_SNAKE_CASE__													= self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

						if pad_token_id is None:
											raise ValueError(
											    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
											    F' padded to `max_length`={max_length}'							)

						SCREAMING_SNAKE_CASE__													= pad_token_id * torch.ones(
						    (tensor.shape[0], max_length)       ,   dtype=tensor.dtype       ,   device=tensor.device							)
						SCREAMING_SNAKE_CASE__													= tensor
						return padded_tensor





