




'''simple docstring'''

import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 						a				(   _lowerCamelCase   ,     unittest.TestCase     ):
				snake_case_      =		RoCBertTokenizer
				snake_case_      =		None
				snake_case_      =		False
				snake_case_      =		True
				snake_case_      =		filter_non_english


				def   A_ (		self    :					Union[str, Any]					):
									super().setUp()

									snake_case_									=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
									snake_case_									=				{}
									snake_case_									=				{}
									for i, value in enumerate(lowercase_					):
														snake_case_									=				i
														snake_case_									=				i
									snake_case_									=				os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['''vocab_file''']					)
									snake_case_									=				os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['''word_shape_file''']					)
									snake_case_									=				os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['''word_pronunciation_file''']					)
									with open(self.vocab_file       ,				'''w'''       ,				encoding='''utf-8'''					) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
									with open(self.word_shape_file       ,				'''w'''       ,				encoding='''utf-8'''					) as word_shape_writer:
														json.dump(lowercase_       ,				lowercase_       ,				ensure_ascii=lowercase_					)
									with open(self.word_pronunciation_file       ,				'''w'''       ,				encoding='''utf-8'''					) as word_pronunciation_writer:
														json.dump(lowercase_       ,				lowercase_       ,				ensure_ascii=lowercase_					)


				def   A_ (		self    :					List[str]					):
									snake_case_									=				self.tokenizer_class(self.vocab_file       ,				self.word_shape_file       ,				self.word_pronunciation_file					)

									snake_case_									=				tokenizer.tokenize('''你好[SEP]你是谁'''					)
									self.assertListEqual(lowercase_       ,				['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']					)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_					)       ,				[5, 6, 2, 5, 7, 8]					)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(lowercase_					)       ,				[5, 6, 2, 5, 7, 8]					)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(lowercase_					)       ,				[5, 6, 2, 5, 7, 8]					)


				def   A_ (		self    :					str					):
									snake_case_									=				RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)       ,				['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)


				def   A_ (		self    :					Tuple					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)       ,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''hello''']					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''h\u00E9llo''']					)


				def   A_ (		self    :					str					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''hello''']					)


				def   A_ (		self    :					List[str]					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)       ,				['''hello''']					)


				def   A_ (		self    :					int					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)       ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)


				def   A_ (		self    :					List[str]					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_       ,				strip_accents=lowercase_					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)       ,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				RoCBertBasicTokenizer(do_lower_case=lowercase_       ,				never_split=['''[UNK]''']					)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)       ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									snake_case_									=				{}
									for i, token in enumerate(lowercase_					):
														snake_case_									=				i
									snake_case_									=				RoCBertWordpieceTokenizer(vocab=lowercase_       ,				unk_token='''[UNK]'''					)

									self.assertListEqual(tokenizer.tokenize(''''''					)       ,				[]					)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)       ,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)       ,				['''[UNK]''', '''runn''', '''##ing''']					)


				def   A_ (		self    :					Optional[int]					):
									self.assertTrue(_is_whitespace(''' '''					)					)
									self.assertTrue(_is_whitespace('''\t'''					)					)
									self.assertTrue(_is_whitespace('''\r'''					)					)
									self.assertTrue(_is_whitespace('''\n'''					)					)
									self.assertTrue(_is_whitespace('''\u00A0'''					)					)

									self.assertFalse(_is_whitespace('''A'''					)					)
									self.assertFalse(_is_whitespace('''-'''					)					)


				def   A_ (		self    :					List[Any]					):
									self.assertTrue(_is_control('''\u0005'''					)					)

									self.assertFalse(_is_control('''A'''					)					)
									self.assertFalse(_is_control(''' '''					)					)
									self.assertFalse(_is_control('''\t'''					)					)
									self.assertFalse(_is_control('''\r'''					)					)


				def   A_ (		self    :					List[str]					):
									self.assertTrue(_is_punctuation('''-'''					)					)
									self.assertTrue(_is_punctuation('''$'''					)					)
									self.assertTrue(_is_punctuation('''`'''					)					)
									self.assertTrue(_is_punctuation('''.'''					)					)

									self.assertFalse(_is_punctuation('''A'''					)					)
									self.assertFalse(_is_punctuation(''' '''					)					)


				def   A_ (		self    :					str					):
									snake_case_									=				self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(lowercase_					) for t in ['''Test''', '''\xad''', '''test''']]       ,				[['''[UNK]'''], [], ['''[UNK]''']]					)

									if self.test_rust_tokenizer:
														snake_case_									=				self.get_rust_tokenizer()
														self.assertListEqual(
														    [rust_tokenizer.tokenize(lowercase_					) for t in ['''Test''', '''\xad''', '''test''']]       ,				[['''[UNK]'''], [], ['''[UNK]''']]					)


				def   A_ (		self    :					Dict					):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"					):
																			snake_case_									=				self.rust_tokenizer_class.from_pretrained(lowercase_       ,				**lowercase_					)

																			snake_case_									=				F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																			snake_case_									=				tokenizer_r.encode_plus(
																			    lowercase_       ,				return_attention_mask=lowercase_       ,				return_token_type_ids=lowercase_       ,				return_offsets_mapping=lowercase_       ,				add_special_tokens=lowercase_       ,				)

																			snake_case_									=				tokenizer_r.do_lower_case if hasattr(lowercase_       ,				'''do_lower_case'''					) else False
																			snake_case_									=				(
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), '''A'''),
																			        ((1, 2), ''','''),
																			        ((3, 5), '''na'''),
																			        ((5, 6), '''##ï'''),
																			        ((6, 8), '''##ve'''),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), '''Allen'''),
																			        ((21, 23), '''##NL'''),
																			        ((23, 24), '''##P'''),
																			        ((25, 33), '''sentence'''),
																			        ((33, 34), '''.'''),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), '''a'''),
																			        ((1, 2), ''','''),
																			        ((3, 8), '''naive'''),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), '''allen'''),
																			        ((21, 23), '''##nl'''),
																			        ((23, 24), '''##p'''),
																			        ((25, 33), '''sentence'''),
																			        ((33, 34), '''.'''),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]       ,				tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
																			self.assertEqual([e[0] for e in expected_results]       ,				tokens['''offset_mapping''']					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				['''的''', '''人''', '''有''']
									snake_case_									=				''''''.join(lowercase_					)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"					):
																			snake_case_									=				True
																			snake_case_									=				self.tokenizer_class.from_pretrained(lowercase_       ,				**lowercase_					)
																			snake_case_									=				self.rust_tokenizer_class.from_pretrained(lowercase_       ,				**lowercase_					)

																			snake_case_									=				tokenizer_p.encode(lowercase_       ,				add_special_tokens=lowercase_					)
																			snake_case_									=				tokenizer_r.encode(lowercase_       ,				add_special_tokens=lowercase_					)

																			snake_case_									=				tokenizer_r.convert_ids_to_tokens(lowercase_					)
																			snake_case_									=				tokenizer_p.convert_ids_to_tokens(lowercase_					)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(lowercase_       ,				lowercase_					)
																			self.assertListEqual(lowercase_       ,				lowercase_					)

																			snake_case_									=				False
																			snake_case_									=				self.rust_tokenizer_class.from_pretrained(lowercase_       ,				**lowercase_					)
																			snake_case_									=				self.tokenizer_class.from_pretrained(lowercase_       ,				**lowercase_					)

																			snake_case_									=				tokenizer_r.encode(lowercase_       ,				add_special_tokens=lowercase_					)
																			snake_case_									=				tokenizer_p.encode(lowercase_       ,				add_special_tokens=lowercase_					)

																			snake_case_									=				tokenizer_r.convert_ids_to_tokens(lowercase_					)
																			snake_case_									=				tokenizer_p.convert_ids_to_tokens(lowercase_					)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			snake_case_									=				[
																			    F"##{token}" if idx != 0 else token for idx, token in enumerate(lowercase_					)
																			]
																			self.assertListEqual(lowercase_       ,				lowercase_					)
																			self.assertListEqual(lowercase_       ,				lowercase_					)


				@slow
				def   A_ (		self    :					Tuple					):
									snake_case_									=				self.tokenizer_class(self.vocab_file       ,				self.word_shape_file       ,				self.word_pronunciation_file					)

									snake_case_									=				tokenizer.encode('''你好'''       ,				add_special_tokens=lowercase_					)
									snake_case_									=				tokenizer.encode('''你是谁'''       ,				add_special_tokens=lowercase_					)

									snake_case_									=				tokenizer.build_inputs_with_special_tokens(lowercase_					)
									snake_case_									=				tokenizer.build_inputs_with_special_tokens(lowercase_       ,				lowercase_					)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


				def   A_ (		self    :					List[Any]					):
									snake_case_									=				self.get_tokenizers(do_lower_case=lowercase_					)
									for tokenizer in tokenizers:
														with self.subTest(F"{tokenizer.__class__.__name__}"					):
																			snake_case_									=				'''你好，你是谁'''
																			snake_case_									=				tokenizer.tokenize(lowercase_					)
																			snake_case_									=				tokenizer.convert_tokens_to_ids(lowercase_					)
																			snake_case_									=				tokenizer.convert_tokens_to_shape_ids(lowercase_					)
																			snake_case_									=				tokenizer.convert_tokens_to_pronunciation_ids(lowercase_					)
																			snake_case_									=				tokenizer.prepare_for_model(
																			    lowercase_       ,				lowercase_       ,				lowercase_       ,				add_special_tokens=lowercase_					)

																			snake_case_									=				tokenizer.encode_plus(lowercase_       ,				add_special_tokens=lowercase_					)

																			self.assertEqual(lowercase_       ,				lowercase_					)









from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
							from fairscale.optim import OSS


A__	:			Tuple			   = logging.get_logger(__name__)

A__	:			int			   = {
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    'constant': get_constant_schedule,
    'constant_w_warmup': get_constant_schedule_with_warmup,
}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				def __init__(     self						:    Optional[int],  lowerCamelCase						:    int=None,  lowerCamelCase						:    int=None,  *lowerCamelCase						:    List[Any],  **lowerCamelCase						:    Any  ):

								'''simple docstring'''
								super().__init__(*lowerCamelCase,  **lowerCamelCase  )

								if config is None:
												assert isinstance(self.model,  lowerCamelCase  ), (
												    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
												    F""" {self.model.__class__}"""
												)
												lowercase__    =			self.model.config
								else:
												lowercase__    =			config

								lowercase__    =			data_args
								lowercase__    =			self.config.tgt_vocab_size if isinstance(self.config,  lowerCamelCase  ) else self.config.vocab_size

								if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
												assert self.config.pad_token_id is not None, (
												    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
												    " calculation or doing label smoothing."
												)

								if self.config.pad_token_id is None and self.config.eos_token_id is not None:
												logger.warning(
												    F"""The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for"""
												    ''' padding..'''  )

								if self.args.label_smoothing == 0:
												lowercase__    =			torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id  )
								else:
												# dynamically import label_smoothed_nll_loss
												from utils import label_smoothed_nll_loss

												lowercase__    =			label_smoothed_nll_loss




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    int  ):

								'''simple docstring'''
								if self.optimizer is None:
												lowercase__    =			['''bias''', '''LayerNorm.weight''']
												lowercase__    =			[
												    {
												        '''params''': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay  )],
												        '''weight_decay''': self.args.weight_decay,
												    },
												    {
												        '''params''': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay  )],
												        '''weight_decay''': 0.0,
												    },
												]
												lowercase__    =			Adafactor if self.args.adafactor else AdamW
												if self.args.adafactor:
																lowercase__    =			Adafactor
																lowercase__    =			{'''scale_parameter''': False, '''relative_step''': False}
												else:
																lowercase__    =			AdamW
																lowercase__    =			{
																    '''betas''': (self.args.adam_betaa, self.args.adam_betaa),
																    '''eps''': self.args.adam_epsilon,
																}
												lowercase__    =			self.args.learning_rate
												if self.sharded_ddp:
																lowercase__    =			OSS(
																    params=lowerCamelCase,  optim=lowerCamelCase,  **lowerCamelCase,  )
												else:
																lowercase__    =			optimizer_cls(lowerCamelCase,  **lowerCamelCase  )

								if self.lr_scheduler is None:
												lowercase__    =			self._get_lr_scheduler(lowerCamelCase  )
								else:  # ignoring --lr_scheduler
												logger.warning('''scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'''  )




				def 				lowercase__   (     self						:    List[str],  lowerCamelCase						:    Optional[int]  ):

								'''simple docstring'''
								lowercase__    =			arg_to_scheduler[self.args.lr_scheduler]
								if self.args.lr_scheduler == "constant":
												lowercase__    =			schedule_func(self.optimizer  )
								elif self.args.lr_scheduler == "constant_w_warmup":
												lowercase__    =			schedule_func(self.optimizer,  num_warmup_steps=self.args.warmup_steps  )
								else:
												lowercase__    =			schedule_func(
												    self.optimizer,  num_warmup_steps=self.args.warmup_steps,  num_training_steps=lowerCamelCase  )
								return scheduler




				def 				lowercase__   (     self						:    List[Any]  ):

								'''simple docstring'''
								if isinstance(self.train_dataset,  torch.utils.data.IterableDataset  ):
												return None
								elif is_torch_tpu_available():
												return get_tpu_sampler(self.train_dataset  )
								else:
												if self.args.sortish_sampler:
																self.train_dataset.make_sortish_sampler(
																    self.args.per_device_train_batch_size,  distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED),  )

												return (
												    RandomSampler(self.train_dataset  )
												    if self.args.local_rank == -1
												    else DistributedSampler(self.train_dataset  )
												)




				def 				lowercase__   (     self						:    Any,  lowerCamelCase						:    List[Any],  lowerCamelCase						:    str,  lowerCamelCase						:    Union[str, Any]  ):

								'''simple docstring'''
								if self.args.label_smoothing == 0:
												if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																# force training to ignore pad token
																lowercase__    =			model(**lowerCamelCase,  use_cache=lowerCamelCase  )[0]
																lowercase__    =			self.loss_fn(logits.view(-1,  logits.shape[-1]  ),  labels.view(-1  )  )
												else:
																# compute usual loss via models
																lowercase__ ,						lowercase__    =			model(**lowerCamelCase,  labels=lowerCamelCase,  use_cache=lowerCamelCase  )[:2]
								else:
												# compute label smoothed loss
												lowercase__    =			model(**lowerCamelCase,  use_cache=lowerCamelCase  )[0]
												lowercase__    =			torch.nn.functional.log_softmax(lowerCamelCase,  dim=-1  )
												lowercase__ ,						lowercase__    =			self.loss_fn(lowerCamelCase,  lowerCamelCase,  self.args.label_smoothing,  ignore_index=self.config.pad_token_id  )
								return loss, logits




				def 				lowercase__   (     self						:    List[str],  lowerCamelCase						:    Optional[Any],  lowerCamelCase						:    Tuple  ):

								'''simple docstring'''
								lowercase__    =			inputs.pop('''labels'''  )
								lowercase__ ,						lowercase__    =			self._compute_loss(lowerCamelCase,  lowerCamelCase,  lowerCamelCase  )
								return loss




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    nn.Module,  lowerCamelCase						:    Dict[str, Union[torch.Tensor, Any]],  lowerCamelCase						:    bool,  lowerCamelCase						:    Optional[List[str]] = None,  ):

								'''simple docstring'''
								lowercase__    =			self._prepare_inputs(lowerCamelCase  )

								lowercase__    =			{
								    '''max_length''': self.data_args.val_max_target_length
								    if self.data_args is not None
								    else self.config.max_length,
								    '''num_beams''': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
								}

								if self.args.predict_with_generate and not self.args.prediction_loss_only:
												lowercase__    =			self.model.generate(
												    inputs['''input_ids'''],  attention_mask=inputs['''attention_mask'''],  **lowerCamelCase,  )
												# in case the batch is shorter than max length, the output should be padded
												if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																lowercase__    =			self._pad_tensors_to_max_len(lowerCamelCase,  gen_kwargs['''max_length''']  )

								lowercase__    =			inputs.pop('''labels'''  )
								with torch.no_grad():
												# compute loss on predict data
												lowercase__ ,						lowercase__    =			self._compute_loss(lowerCamelCase,  lowerCamelCase,  lowerCamelCase  )

								lowercase__    =			loss.mean().detach()
								if self.args.prediction_loss_only:
												return (loss, None, None)

								lowercase__    =			generated_tokens if self.args.predict_with_generate else logits

								if labels.shape[-1] < gen_kwargs["max_length"]:
												lowercase__    =			self._pad_tensors_to_max_len(lowerCamelCase,  gen_kwargs['''max_length''']  )

								return (loss, logits, labels)




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    str,  lowerCamelCase						:    Any  ):

								'''simple docstring'''
								# If PAD token is not defined at least EOS token has to be defined
								lowercase__    =			self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

								if pad_token_id is None:
												raise ValueError(
												    '''Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'''
												    F""" padded to `max_length`={max_length}"""  )

								lowercase__    =			pad_token_id * torch.ones(
								    (tensor.shape[0], max_length),  dtype=tensor.dtype,  device=tensor.device  )
								lowercase__    =			tensor
								return padded_tensor




