


"""simple docstring"""
import unittest

from transformers import SqueezeBertConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
			    SqueezeBertForMaskedLM,
			    SqueezeBertForMultipleChoice,
			    SqueezeBertForQuestionAnswering,
			    SqueezeBertForSequenceClassification,
			    SqueezeBertForTokenClassification,
			    SqueezeBertModel,
			)



class 			_UpperCAmelCase (							UpperCamelCase_				):






		'''simple docstring'''

		def __init__(self				,       a_				,       a_=13				,       a_=7				,       a_=True				,       a_=True				,       a_=False				,       a_=True				,       a_=99				,       a_=32				,       a_=5				,       a_=4				,       a_=64				,       a_="gelu"				,       a_=0.1				,       a_=0.1				,       a_=5_12				,       a_=16				,       a_=2				,       a_=0.02				,       a_=3				,       a_=4				,       a_=None				,       a_=2				,       a_=2				,       a_=2				,       a_=2				,       a_=4				,       a_=1				,       ):





			'''simple docstring'''

			__snake_case	:						str						=				parent
			__snake_case	:						Optional[Any]						=				batch_size
			__snake_case	:						Union[str, Any]						=				seq_length
			__snake_case	:						Optional[int]						=				is_training
			__snake_case	:						Dict						=				use_input_mask
			__snake_case	:						str						=				use_token_type_ids
			__snake_case	:						List[str]						=				use_labels
			__snake_case	:						Optional[Any]						=				vocab_size
			__snake_case	:						Dict						=				hidden_size
			__snake_case	:						List[Any]						=				num_hidden_layers
			__snake_case	:						Optional[Any]						=				num_attention_heads
			__snake_case	:						Union[str, Any]						=				intermediate_size
			__snake_case	:						List[str]						=				hidden_act
			__snake_case	:						str						=				hidden_dropout_prob
			__snake_case	:						List[Any]						=				attention_probs_dropout_prob
			__snake_case	:						Optional[Any]						=				max_position_embeddings
			__snake_case	:						int						=				type_vocab_size
			__snake_case	:						Any						=				type_sequence_label_size
			__snake_case	:						List[Any]						=				initializer_range
			__snake_case	:						Optional[Any]						=				num_labels
			__snake_case	:						int						=				num_choices
			__snake_case	:						Optional[int]						=				scope
			__snake_case	:						Union[str, Any]						=				q_groups
			__snake_case	:						List[str]						=				k_groups
			__snake_case	:						Tuple						=				v_groups
			__snake_case	:						Tuple						=				post_attention_groups
			__snake_case	:						Optional[Any]						=				intermediate_groups
			__snake_case	:						List[str]						=				output_groups



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						str						=				ids_tensor([self.batch_size, self.seq_length]				,       self.vocab_size					)

			__snake_case	:						Optional[int]						=				None
			if self.use_input_mask:
				__snake_case	:						Optional[Any]						=				random_attention_mask([self.batch_size, self.seq_length]					)

			__snake_case	:						Optional[Any]						=				None
			__snake_case	:						Any						=				None
			__snake_case	:						List[Any]						=				None
			if self.use_labels:
				__snake_case	:						Optional[Any]						=				ids_tensor([self.batch_size]				,       self.type_sequence_label_size					)
				__snake_case	:						Optional[int]						=				ids_tensor([self.batch_size, self.seq_length]				,       self.num_labels					)
				__snake_case	:						Union[str, Any]						=				ids_tensor([self.batch_size]				,       self.num_choices					)

			__snake_case	:						Tuple						=				self.get_config()

			return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			return SqueezeBertConfig(
			    embedding_size=self.hidden_size				,       vocab_size=self.vocab_size				,       hidden_size=self.hidden_size				,       num_hidden_layers=self.num_hidden_layers				,       num_attention_heads=self.num_attention_heads				,       intermediate_size=self.intermediate_size				,       hidden_act=self.hidden_act				,       attention_probs_dropout_prob=self.hidden_dropout_prob				,       attention_dropout=self.attention_probs_dropout_prob				,       max_position_embeddings=self.max_position_embeddings				,       initializer_range=self.initializer_range				,       q_groups=self.q_groups				,       k_groups=self.k_groups				,       v_groups=self.v_groups				,       post_attention_groups=self.post_attention_groups				,       intermediate_groups=self.intermediate_groups				,       output_groups=self.output_groups				,       )



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_				,       a_				,       a_				,       a_				,       a_					):





			'''simple docstring'''

			__snake_case	:						List[Any]						=				SqueezeBertModel(config=lowercase_					)
			model.to(lowercase_					)
			model.eval()
			__snake_case	:						Tuple						=				model(lowercase_				,       lowercase_					)
			__snake_case	:						List[Any]						=				model(lowercase_					)
			self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.hidden_size)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_				,       a_				,       a_				,       a_				,       a_					):





			'''simple docstring'''

			__snake_case	:						Optional[Any]						=				SqueezeBertForMaskedLM(config=lowercase_					)
			model.to(lowercase_					)
			model.eval()
			__snake_case	:						int						=				model(lowercase_				,       attention_mask=lowercase_				,       labels=lowercase_					)
			self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.vocab_size)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_				,       a_				,       a_				,       a_				,       a_					):





			'''simple docstring'''

			__snake_case	:						Union[str, Any]						=				SqueezeBertForQuestionAnswering(config=lowercase_					)
			model.to(lowercase_					)
			model.eval()
			__snake_case	:						Optional[Any]						=				model(
			    lowercase_				,       attention_mask=lowercase_				,       start_positions=lowercase_				,       end_positions=lowercase_					)
			self.parent.assertEqual(result.start_logits.shape				,       (self.batch_size, self.seq_length)					)
			self.parent.assertEqual(result.end_logits.shape				,       (self.batch_size, self.seq_length)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_				,       a_				,       a_				,       a_				,       a_					):





			'''simple docstring'''

			__snake_case	:						int						=				self.num_labels
			__snake_case	:						Any						=				SqueezeBertForSequenceClassification(lowercase_					)
			model.to(lowercase_					)
			model.eval()
			__snake_case	:						Tuple						=				model(lowercase_				,       attention_mask=lowercase_				,       labels=lowercase_					)
			self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.num_labels)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_				,       a_				,       a_				,       a_				,       a_					):





			'''simple docstring'''

			__snake_case	:						Any						=				self.num_labels
			__snake_case	:						str						=				SqueezeBertForTokenClassification(config=lowercase_					)
			model.to(lowercase_					)
			model.eval()

			__snake_case	:						Any						=				model(lowercase_				,       attention_mask=lowercase_				,       labels=lowercase_					)
			self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.num_labels)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_				,       a_				,       a_				,       a_				,       a_					):





			'''simple docstring'''

			__snake_case	:						Tuple						=				self.num_choices
			__snake_case	:						List[Any]						=				SqueezeBertForMultipleChoice(config=lowercase_					)
			model.to(lowercase_					)
			model.eval()
			__snake_case	:						Dict						=				input_ids.unsqueeze(1					).expand(-1				,       self.num_choices				,       -1					).contiguous()
			__snake_case	:						Union[str, Any]						=				input_mask.unsqueeze(1					).expand(-1				,       self.num_choices				,       -1					).contiguous()
			__snake_case	:						Tuple						=				model(
			    lowercase_				,       attention_mask=lowercase_				,       labels=lowercase_				,       )
			self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.num_choices)					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Dict						=				self.prepare_config_and_inputs()
			(__snake_case)	:						Tuple						=				config_and_inputs
			__snake_case	:						Dict						=				{'input_ids': input_ids, 'attention_mask': input_mask}
			return config, inputs_dict



@require_torch
class 			_UpperCAmelCase (							UpperCamelCase_,				UpperCamelCase_,				unittest.TestCase				):






		'''simple docstring'''

		lowerCamelCase__				=(
		    (
		        SqueezeBertModel,
		        SqueezeBertForMaskedLM,
		        SqueezeBertForMultipleChoice,
		        SqueezeBertForQuestionAnswering,
		        SqueezeBertForSequenceClassification,
		        SqueezeBertForTokenClassification,
		    )
		    if is_torch_available()
		    else None
		)
		lowerCamelCase__				=(
		    {
		        'feature-extraction': SqueezeBertModel,
		        'fill-mask': SqueezeBertForMaskedLM,
		        'question-answering': SqueezeBertForQuestionAnswering,
		        'text-classification': SqueezeBertForSequenceClassification,
		        'token-classification': SqueezeBertForTokenClassification,
		        'zero-shot': SqueezeBertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		lowerCamelCase__				=False
		lowerCamelCase__				=True
		lowerCamelCase__				=False



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						str						=				SqueezeBertModelTester(self					)
			__snake_case	:						Dict						=				ConfigTester(self				,       config_class=lowercase_				,       dim=37					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			self.config_tester.run_common_tests()



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[Any]						=				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_squeezebert_model(*lowercase_					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Dict						=				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_squeezebert_for_masked_lm(*lowercase_					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						int						=				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_squeezebert_for_question_answering(*lowercase_					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Union[str, Any]						=				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_squeezebert_for_sequence_classification(*lowercase_					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Tuple						=				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_squeezebert_for_token_classification(*lowercase_					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Union[str, Any]						=				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_squeezebert_for_multiple_choice(*lowercase_					)



		@slow
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			for model_name in SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
				__snake_case	:						List[Any]						=				SqueezeBertModel.from_pretrained(lowercase_					)
				self.assertIsNotNone(lowercase_					)



@require_sentencepiece
@require_tokenizers
@require_torch
class 			_UpperCAmelCase (							unittest.TestCase				):






		'''simple docstring'''

		@slow
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Tuple						=				SqueezeBertForSequenceClassification.from_pretrained('''squeezebert/squeezebert-mnli'''					)

			__snake_case	:						Optional[Any]						=				torch.tensor([[1, 2_94_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 13, 15_88, 2]]					)
			__snake_case	:						Optional[Any]						=				model(lowercase_					)[0]
			__snake_case	:						int						=				torch.Size((1, 3)					)
			self.assertEqual(output.shape				,       lowercase_					)
			__snake_case	:						Optional[Any]						=				torch.tensor([[0.6401, -0.0349, -0.6041]]					)
			self.assertTrue(torch.allclose(lowercase_				,       lowercase_				,       atol=1E-4					)					)









import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase											=       get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCamelCase											=       5_00_03
lowerCamelCase											=       5_00_02




@require_sentencepiece
@require_tokenizers
class A  (   UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  int      =PLBartTokenizer
      UpperCamelCase__							:  Dict      =None
      UpperCamelCase__							:  Optional[Any]      =False






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						List[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase       :		Any		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )
             tokenizer.save_pretrained(self.tmpdirname )






      def        lowerCamelCase       (					self						:     List[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Optional[int]		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		str		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		List[Any]		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )

             _lowerCamelCase       :		str		=tokenizer.vocab_size
             _lowerCamelCase       :		List[str]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 4			,   lowercase_ )]

             self.assertListEqual(lowercase_			,   ['__java__', '__python__', '__en_XX__', '<mask>'] )

             _lowerCamelCase       :		Optional[Any]		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Dict		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )






      def        lowerCamelCase       (					self						:     str )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=PLBartTokenizer(lowercase_			,   language_codes='multi'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Any		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.vocab_size
             _lowerCamelCase       :		Optional[int]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 7			,   lowercase_ )]

             self.assertListEqual(
                 lowercase_			,   ['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__'] )
             _lowerCamelCase       :		int		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Any		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )




@require_torch
@require_sentencepiece
@require_tokenizers
class A  (   unittest.TestCase     ):
      UpperCamelCase__							:  List[Any]      ='uclanlp/plbart-python-en_XX'
      UpperCamelCase__							:  List[str]      =[
          'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
          'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
      ]
      UpperCamelCase__							:  Optional[int]      =[
          'Returns the maximum value of a b c.',
          'Sums the values of a b c.',
      ]
      UpperCamelCase__							:  str      =[
          134,
          5452,
          33460,
          33441,
          33463,
          33465,
          33463,
          33449,
          988,
          20,
          33456,
          19,
          33456,
          771,
          39,
          4258,
          889,
          3318,
          33441,
          33463,
          33465,
          33463,
          33449,
          2471,
          2,
          PYTHON_CODE,
      ]






      @classmethod
      def        lowerCamelCase       (					cls						:     str )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase       :		PLBartTokenizer		=PLBartTokenizer.from_pretrained(
                 cls.checkpoint_name			,   language_codes='base'			,   src_lang='python'			,   tgt_lang='en_XX' )
             _lowerCamelCase       :		Any		=1
             return cls






      def        lowerCamelCase       (					self						:     List[str] )					->						Any:
             """simple docstring"""




             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']			,   5_0001 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']			,   5_0002 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']			,   5_0003 )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Tuple:
             """simple docstring"""




             self.assertIn(lowercase_			,   self.tokenizer.all_special_ids )
             _lowerCamelCase       :		Dict		=[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
             _lowerCamelCase       :		Optional[int]		=self.tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_ )
             _lowerCamelCase       :		int		=self.tokenizer.decode(generated_ids[1:]			,   skip_special_tokens=lowercase_ )
             self.assertEqual(lowercase_			,   lowercase_ )
             self.assertNotIn(self.tokenizer.eos_token			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
             self.assertIsInstance(src_text[0]			,   lowercase_ )
             _lowerCamelCase       :		Tuple		=10
             _lowerCamelCase       :		Optional[int]		=self.tokenizer(lowercase_			,   max_length=lowercase_			,   truncation=lowercase_ ).input_ids[0]
             self.assertEqual(ids[-2]			,   2 )
             self.assertEqual(ids[-1]			,   lowercase_ )
             self.assertEqual(len(lowercase_ )			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Dict )					->						Optional[int]:
             """simple docstring"""




             self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__'] )			,   [5_0004, 5_0001] )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=tempfile.mkdtemp()
             _lowerCamelCase       :		Dict		=self.tokenizer.fairseq_tokens_to_ids
             self.tokenizer.save_pretrained(lowercase_ )
             _lowerCamelCase       :		Any		=PLBartTokenizer.from_pretrained(lowercase_ )
             self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,   lowercase_ )






      @require_torch
      def        lowerCamelCase       (					self						:     Optional[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=self.tokenizer(self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   return_tensors='pt' )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
             self.assertEqual(batch.input_ids[1][-2:].tolist()			,   [2, PYTHON_CODE] )
             self.assertEqual(batch.decoder_input_ids[1][0]			,   lowercase_ )
             self.assertEqual(batch.decoder_input_ids[1][-1]			,   2 )
             self.assertEqual(batch.labels[1][-2:].tolist()			,   [2, EN_CODE] )






      @require_torch
      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		int		=self.tokenizer(
                 self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=len(self.expected_src_tokens )			,   return_tensors='pt'			,   )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             self.assertIsInstance(lowercase_			,   lowercase_ )

             self.assertEqual((2, 26)			,   batch.input_ids.shape )
             self.assertEqual((2, 26)			,   batch.attention_mask.shape )
             _lowerCamelCase       :		List[Any]		=batch.input_ids.tolist()[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )
             self.assertEqual(2			,   batch.decoder_input_ids[0, -1] )  # EOS
             # Test that special tokens are reset
             self.assertEqual(self.tokenizer.prefix_tokens			,   [] )
             self.assertEqual(self.tokenizer.suffix_tokens			,   [self.tokenizer.eos_token_id, PYTHON_CODE] )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=self.tokenizer(self.src_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=3			,   return_tensors='pt' )
             _lowerCamelCase       :		Dict		=self.tokenizer(
                 text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=10			,   return_tensors='pt' )
             _lowerCamelCase       :		List[str]		=targets['input_ids']
             _lowerCamelCase       :		Optional[Any]		=shift_tokens_right(lowercase_			,   self.tokenizer.pad_token_id )

             self.assertEqual(batch.input_ids.shape[1]			,   3 )
             self.assertEqual(batch.decoder_input_ids.shape[1]			,   10 )






      @require_torch
      def        lowerCamelCase       (					self						:     Any )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Any		=self.tokenizer._build_translation_inputs(
                 'A test'			,   return_tensors='pt'			,   src_lang='en_XX'			,   tgt_lang='java' )

             self.assertEqual(
                 nested_simplify(lowercase_ )			,   {
                     # A, test, EOS, en_XX
                     'input_ids': [[150, 242, 2, 5_0003]],
                     'attention_mask': [[1, 1, 1, 1]],
                     # java
                     'forced_bos_token_id': 5_0001,
                 }			,   )


