


import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import ASTForAudioClassification, ASTModel
 from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
     AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
 )


if is_torchaudio_available():
 import torchaudio

 from transformers import ASTFeatureExtractor




class A  :






      def __init__(					self						:     Any			,   lowercase_						:     Union[str, Any]			,   lowercase_						:     Dict=13			,   lowercase_						:     Dict=2			,   lowercase_						:     str=24			,   lowercase_						:     str=16			,   lowercase_						:     Dict=True			,   lowercase_						:     Optional[int]=True			,   lowercase_						:     Dict=32			,   lowercase_						:     Tuple=5			,   lowercase_						:     Optional[Any]=4			,   lowercase_						:     Tuple=37			,   lowercase_						:     Optional[Any]="gelu"			,   lowercase_						:     str=0.1			,   lowercase_						:     List[str]=0.1			,   lowercase_						:     int=10			,   lowercase_						:     Optional[Any]=0.02			,   lowercase_						:     Union[str, Any]=None			,   lowercase_						:     List[Any]=2			,   lowercase_						:     Optional[Any]=2			,   )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=parent
             _lowerCamelCase       :		List[Any]		=batch_size
             _lowerCamelCase       :		List[Any]		=patch_size
             _lowerCamelCase       :		int		=max_length
             _lowerCamelCase       :		int		=num_mel_bins
             _lowerCamelCase       :		Optional[int]		=is_training
             _lowerCamelCase       :		Any		=use_labels
             _lowerCamelCase       :		List[Any]		=hidden_size
             _lowerCamelCase       :		Union[str, Any]		=num_hidden_layers
             _lowerCamelCase       :		Any		=num_attention_heads
             _lowerCamelCase       :		Optional[Any]		=intermediate_size
             _lowerCamelCase       :		Optional[Any]		=hidden_act
             _lowerCamelCase       :		List[Any]		=hidden_dropout_prob
             _lowerCamelCase       :		List[str]		=attention_probs_dropout_prob
             _lowerCamelCase       :		List[str]		=type_sequence_label_size
             _lowerCamelCase       :		Dict		=initializer_range
             _lowerCamelCase       :		List[Any]		=scope
             _lowerCamelCase       :		Dict		=frequency_stride
             _lowerCamelCase       :		Optional[int]		=time_stride

             # in AST, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
             _lowerCamelCase       :		Optional[int]		=(self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
             _lowerCamelCase       :		Any		=(self.max_length - self.patch_size) // self.time_stride + 1
             _lowerCamelCase       :		Optional[Any]		=frequency_out_dimension * time_out_dimension
             _lowerCamelCase       :		List[str]		=num_patches + 2






      def        lowerCamelCase       (					self						:     int )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		Any		=floats_tensor([self.batch_size, self.max_length, self.num_mel_bins] )

             _lowerCamelCase       :		Dict		=None
             if self.use_labels:
                    _lowerCamelCase       :		Optional[Any]		=ids_tensor([self.batch_size]			,   self.type_sequence_label_size )

             _lowerCamelCase       :		Tuple		=self.get_config()

             return config, input_values, labels






      def        lowerCamelCase       (					self						:     Dict )					->						Optional[Any]:
             """simple docstring"""




             return ASTConfig(
                 patch_size=self.patch_size			,   max_length=self.max_length			,   num_mel_bins=self.num_mel_bins			,   hidden_size=self.hidden_size			,   num_hidden_layers=self.num_hidden_layers			,   num_attention_heads=self.num_attention_heads			,   intermediate_size=self.intermediate_size			,   hidden_act=self.hidden_act			,   hidden_dropout_prob=self.hidden_dropout_prob			,   attention_probs_dropout_prob=self.attention_probs_dropout_prob			,   is_decoder=lowercase_			,   initializer_range=self.initializer_range			,   frequency_stride=self.frequency_stride			,   time_stride=self.time_stride			,   )






      def        lowerCamelCase       (					self						:     Tuple			,   lowercase_						:     List[Any]			,   lowercase_						:     int			,   lowercase_						:     Optional[int] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		int		=ASTModel(config=lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		Tuple		=model(lowercase_ )
             self.parent.assertEqual(result.last_hidden_state.shape			,   (self.batch_size, self.seq_length, self.hidden_size) )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		Dict		=self.prepare_config_and_inputs()
             (
                 (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     
             )       :		Dict		=config_and_inputs
             _lowerCamelCase       :		Tuple		={'input_values': input_values}
             return config, inputs_dict




@require_torch
class A  (   UpperCamelCase_    ,							UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  Dict      =(
          (
              ASTModel,
              ASTForAudioClassification,
          )
          if is_torch_available()
          else ()
      )
      UpperCamelCase__							:  str      =(
          {'audio-classification': ASTForAudioClassification, 'feature-extraction': ASTModel}
          if is_torch_available()
          else {}
      )
      UpperCamelCase__							:  Any      =False
      UpperCamelCase__							:  str      =False
      UpperCamelCase__							:  Union[str, Any]      =False
      UpperCamelCase__							:  Optional[Any]      =False






      def        lowerCamelCase       (					self						:     Dict			,   lowercase_						:     List[Any]			,   lowercase_						:     List[str]			,   lowercase_						:     Union[str, Any]			,   lowercase_						:     str			,   lowercase_						:     Optional[int] )					->						List[str]:
             """simple docstring"""




             if pipeline_test_casse_name == "AudioClassificationPipelineTests":
                    return True

             return False






      def        lowerCamelCase       (					self						:     Dict )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=ASTModelTester(self )
             _lowerCamelCase       :		Optional[int]		=ConfigTester(self			,   config_class=lowercase_			,   has_text_modality=lowercase_			,   hidden_size=37 )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             self.config_tester.run_common_tests()






      @unittest.skip(reason='AST does not use inputs_embeds' )
      def        lowerCamelCase       (					self						:     str )					->						List[Any]:
             """simple docstring"""




             pass






      def        lowerCamelCase       (					self						:     Optional[int] )					->						Any:
             """simple docstring"""




             _lowerCamelCase	,     _lowerCamelCase       :		Optional[Any]		=self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                    _lowerCamelCase       :		int		=model_class(lowercase_ )
                    self.assertIsInstance(model.get_input_embeddings()			,   (nn.Module) )
                    _lowerCamelCase       :		Optional[Any]		=model.get_output_embeddings()
                    self.assertTrue(x is None or isinstance(lowercase_			,   nn.Linear ) )






      def        lowerCamelCase       (					self						:     List[Any] )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase	,     _lowerCamelCase       :		Tuple		=self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                    _lowerCamelCase       :		Tuple		=model_class(lowercase_ )
                    _lowerCamelCase       :		Tuple		=inspect.signature(model.forward )
                    # signature.parameters is an OrderedDict => so arg_names order is deterministic
                    _lowerCamelCase       :		Any		=[*signature.parameters.keys()]

                    _lowerCamelCase       :		List[Any]		=['input_values']
                    self.assertListEqual(arg_names[:1]			,   lowercase_ )






      def        lowerCamelCase       (					self						:     int )					->						List[Any]:
             """simple docstring"""




             _lowerCamelCase       :		int		=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*lowercase_ )






      @slow
      def        lowerCamelCase       (					self						:     Optional[int] )					->						Dict:
             """simple docstring"""




             for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    _lowerCamelCase       :		Union[str, Any]		=ASTModel.from_pretrained(lowercase_ )
                    self.assertIsNotNone(lowercase_ )




def 	a_      (    ):
       '''simple docstring'''





       _lowerCamelCase       :		Optional[int]		=hf_hub_download(
           repo_id='nielsr/audio-spectogram-transformer-checkpoint'		,			filename='sample_audio.flac'		,			repo_type='dataset'     )

       _lowerCamelCase	,     _lowerCamelCase       :		Tuple		=torchaudio.load(SCREAMING_SNAKE_CASE__     )

       return audio, sampling_rate




@require_torch
@require_torchaudio
class A  (   unittest.TestCase     ):






      @cached_property
      def        lowerCamelCase       (					self						:     List[str] )					->						Optional[Any]:
             """simple docstring"""




             return (
                 ASTFeatureExtractor.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593' )
                 if is_torchaudio_available()
                 else None
             )






      @slow
      def        lowerCamelCase       (					self						:     List[Any] )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=self.default_feature_extractor
             _lowerCamelCase       :		List[Any]		=ASTForAudioClassification.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593' ).to(lowercase_ )

             _lowerCamelCase       :		Tuple		=self.default_feature_extractor
             _lowerCamelCase	,     _lowerCamelCase       :		str		=prepare_audio()
             _lowerCamelCase       :		Dict		=audio.squeeze().numpy()
             _lowerCamelCase       :		Optional[Any]		=feature_extractor(lowercase_			,   sampling_rate=lowercase_			,   return_tensors='pt' ).to(lowercase_ )

             # forward pass
             with torch.no_grad():
                    _lowerCamelCase       :		Union[str, Any]		=model(**lowercase_ )

             # verify the logits
             _lowerCamelCase       :		Dict		=torch.Size((1, 527) )
             self.assertEqual(outputs.logits.shape			,   lowercase_ )

             _lowerCamelCase       :		str		=torch.tensor([-0.8760, -7.0042, -8.6602] ).to(lowercase_ )

             self.assertTrue(torch.allclose(outputs.logits[0, :3]			,   lowercase_			,   atol=1E-4 ) )






import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase											=       get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCamelCase											=       5_00_03
lowerCamelCase											=       5_00_02




@require_sentencepiece
@require_tokenizers
class A  (   UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  int      =PLBartTokenizer
      UpperCamelCase__							:  Dict      =None
      UpperCamelCase__							:  Optional[Any]      =False






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						List[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase       :		Any		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )
             tokenizer.save_pretrained(self.tmpdirname )






      def        lowerCamelCase       (					self						:     List[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Optional[int]		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		str		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		List[Any]		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )

             _lowerCamelCase       :		str		=tokenizer.vocab_size
             _lowerCamelCase       :		List[str]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 4			,   lowercase_ )]

             self.assertListEqual(lowercase_			,   ['__java__', '__python__', '__en_XX__', '<mask>'] )

             _lowerCamelCase       :		Optional[Any]		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Dict		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )






      def        lowerCamelCase       (					self						:     str )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=PLBartTokenizer(lowercase_			,   language_codes='multi'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Any		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.vocab_size
             _lowerCamelCase       :		Optional[int]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 7			,   lowercase_ )]

             self.assertListEqual(
                 lowercase_			,   ['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__'] )
             _lowerCamelCase       :		int		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Any		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )




@require_torch
@require_sentencepiece
@require_tokenizers
class A  (   unittest.TestCase     ):
      UpperCamelCase__							:  List[Any]      ='uclanlp/plbart-python-en_XX'
      UpperCamelCase__							:  List[str]      =[
          'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
          'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
      ]
      UpperCamelCase__							:  Optional[int]      =[
          'Returns the maximum value of a b c.',
          'Sums the values of a b c.',
      ]
      UpperCamelCase__							:  str      =[
          134,
          5452,
          33460,
          33441,
          33463,
          33465,
          33463,
          33449,
          988,
          20,
          33456,
          19,
          33456,
          771,
          39,
          4258,
          889,
          3318,
          33441,
          33463,
          33465,
          33463,
          33449,
          2471,
          2,
          PYTHON_CODE,
      ]






      @classmethod
      def        lowerCamelCase       (					cls						:     str )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase       :		PLBartTokenizer		=PLBartTokenizer.from_pretrained(
                 cls.checkpoint_name			,   language_codes='base'			,   src_lang='python'			,   tgt_lang='en_XX' )
             _lowerCamelCase       :		Any		=1
             return cls






      def        lowerCamelCase       (					self						:     List[str] )					->						Any:
             """simple docstring"""




             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']			,   5_0001 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']			,   5_0002 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']			,   5_0003 )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Tuple:
             """simple docstring"""




             self.assertIn(lowercase_			,   self.tokenizer.all_special_ids )
             _lowerCamelCase       :		Dict		=[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
             _lowerCamelCase       :		Optional[int]		=self.tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_ )
             _lowerCamelCase       :		int		=self.tokenizer.decode(generated_ids[1:]			,   skip_special_tokens=lowercase_ )
             self.assertEqual(lowercase_			,   lowercase_ )
             self.assertNotIn(self.tokenizer.eos_token			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
             self.assertIsInstance(src_text[0]			,   lowercase_ )
             _lowerCamelCase       :		Tuple		=10
             _lowerCamelCase       :		Optional[int]		=self.tokenizer(lowercase_			,   max_length=lowercase_			,   truncation=lowercase_ ).input_ids[0]
             self.assertEqual(ids[-2]			,   2 )
             self.assertEqual(ids[-1]			,   lowercase_ )
             self.assertEqual(len(lowercase_ )			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Dict )					->						Optional[int]:
             """simple docstring"""




             self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__'] )			,   [5_0004, 5_0001] )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=tempfile.mkdtemp()
             _lowerCamelCase       :		Dict		=self.tokenizer.fairseq_tokens_to_ids
             self.tokenizer.save_pretrained(lowercase_ )
             _lowerCamelCase       :		Any		=PLBartTokenizer.from_pretrained(lowercase_ )
             self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,   lowercase_ )






      @require_torch
      def        lowerCamelCase       (					self						:     Optional[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=self.tokenizer(self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   return_tensors='pt' )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
             self.assertEqual(batch.input_ids[1][-2:].tolist()			,   [2, PYTHON_CODE] )
             self.assertEqual(batch.decoder_input_ids[1][0]			,   lowercase_ )
             self.assertEqual(batch.decoder_input_ids[1][-1]			,   2 )
             self.assertEqual(batch.labels[1][-2:].tolist()			,   [2, EN_CODE] )






      @require_torch
      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		int		=self.tokenizer(
                 self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=len(self.expected_src_tokens )			,   return_tensors='pt'			,   )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             self.assertIsInstance(lowercase_			,   lowercase_ )

             self.assertEqual((2, 26)			,   batch.input_ids.shape )
             self.assertEqual((2, 26)			,   batch.attention_mask.shape )
             _lowerCamelCase       :		List[Any]		=batch.input_ids.tolist()[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )
             self.assertEqual(2			,   batch.decoder_input_ids[0, -1] )  # EOS
             # Test that special tokens are reset
             self.assertEqual(self.tokenizer.prefix_tokens			,   [] )
             self.assertEqual(self.tokenizer.suffix_tokens			,   [self.tokenizer.eos_token_id, PYTHON_CODE] )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=self.tokenizer(self.src_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=3			,   return_tensors='pt' )
             _lowerCamelCase       :		Dict		=self.tokenizer(
                 text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=10			,   return_tensors='pt' )
             _lowerCamelCase       :		List[str]		=targets['input_ids']
             _lowerCamelCase       :		Optional[Any]		=shift_tokens_right(lowercase_			,   self.tokenizer.pad_token_id )

             self.assertEqual(batch.input_ids.shape[1]			,   3 )
             self.assertEqual(batch.decoder_input_ids.shape[1]			,   10 )






      @require_torch
      def        lowerCamelCase       (					self						:     Any )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Any		=self.tokenizer._build_translation_inputs(
                 'A test'			,   return_tensors='pt'			,   src_lang='en_XX'			,   tgt_lang='java' )

             self.assertEqual(
                 nested_simplify(lowercase_ )			,   {
                     # A, test, EOS, en_XX
                     'input_ids': [[150, 242, 2, 5_0003]],
                     'attention_mask': [[1, 1, 1, 1]],
                     # java
                     'forced_bos_token_id': 5_0001,
                 }			,   )


