code: 
  
  
  import os
  from shutil import copyfile
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  if TYPE_CHECKING:
  	from transformers.pipelines.conversational import Conversation
  
  A_    :					str             =logging.get_logger(__name__)
  
  A_    :					Union[str, Any]             ={"""vocab_file""": """tokenizer.model"""}
  
  A_    :					Any             ={
      """vocab_file""": {
          """hf-internal-testing/llama-tokenizer""": """https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model""",
      },
      """tokenizer_file""": {
          """hf-internal-testing/llama-tokenizer""": """https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json""",
      },
  }
  A_    :					Optional[int]             ={
      """hf-internal-testing/llama-tokenizer""": 2048,
  }
  A_    :					Optional[Any]             ="""▁"""
  
  A_       ,	A_    :					Optional[Any]             ="""[INST]""", """[/INST]"""
  A_       ,	A_    :					Optional[int]             ="""<<SYS>>\n""", """\n<</SYS>>\n\n"""
  
  # fmt: off
  A_    :					Optional[int]             ="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
  answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
  that your responses are socially unbiased and positive in nature.
  
  If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
  correct. If you don't know the answer to a question, please don't share false information."""
  
  
  
  
  
  class        lowercase_				( UpperCamelCase__):
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							snake_case_        =		VOCAB_FILES_NAMES
  							snake_case_        =		PRETRAINED_VOCAB_FILES_MAP
  							snake_case_        =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  							snake_case_        =		['''input_ids''', '''attention_mask''']
  
  
  							def __init__(   self			,    _UpperCAmelCase			,    _UpperCAmelCase="<unk>"			,    _UpperCAmelCase="<s>"			,    _UpperCAmelCase="</s>"			,    _UpperCAmelCase=None			,    _UpperCAmelCase = None			,    _UpperCAmelCase=True			,    _UpperCAmelCase=False			,    _UpperCAmelCase=False			,    _UpperCAmelCase=True			,    **_UpperCAmelCase			,    ):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		{} if sp_model_kwargs is None else sp_model_kwargs
  													a_					=		AddedToken(_UpperCAmelCase			,    lstrip=_UpperCAmelCase			,    rstrip=_UpperCAmelCase		) if isinstance(_UpperCAmelCase			,    _UpperCAmelCase		) else bos_token
  													a_					=		AddedToken(_UpperCAmelCase			,    lstrip=_UpperCAmelCase			,    rstrip=_UpperCAmelCase		) if isinstance(_UpperCAmelCase			,    _UpperCAmelCase		) else eos_token
  													a_					=		AddedToken(_UpperCAmelCase			,    lstrip=_UpperCAmelCase			,    rstrip=_UpperCAmelCase		) if isinstance(_UpperCAmelCase			,    _UpperCAmelCase		) else unk_token
  													a_					=		AddedToken(_UpperCAmelCase			,    lstrip=_UpperCAmelCase			,    rstrip=_UpperCAmelCase		) if isinstance(_UpperCAmelCase			,    _UpperCAmelCase		) else pad_token
  													super().__init__(
  													    bos_token=_UpperCAmelCase			,    eos_token=_UpperCAmelCase			,    unk_token=_UpperCAmelCase			,    pad_token=_UpperCAmelCase			,    add_bos_token=_UpperCAmelCase			,    add_eos_token=_UpperCAmelCase			,    sp_model_kwargs=self.sp_model_kwargs			,    clean_up_tokenization_spaces=_UpperCAmelCase			,    legacy=_UpperCAmelCase			,    **_UpperCAmelCase			,    )
  													if legacy:
  																			logger.warning_once(
  																			    f"You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"
  																			    """ read the related pull request available at https://github.com/huggingface/transformers/pull/24565"""		)
  													a_					=		legacy
  													a_					=		vocab_file
  													a_					=		add_bos_token
  													a_					=		add_eos_token
  													a_					=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
  													self.sp_model.Load(_UpperCAmelCase		)
  
  
  							def __getstate__(   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		self.__dict__.copy()
  													a_					=		None
  													a_					=		self.sp_model.serialized_model_proto()
  													return state
  
  
  							def __setstate__(   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		d
  													a_					=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
  													self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)
  
  
  							@property
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													return self.sp_model.get_piece_size()
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		{self.convert_ids_to_tokens(_UpperCAmelCase		): i for i in range(self.vocab_size		)}
  													vocab.update(self.added_tokens_encoder		)
  													return vocab
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    **_UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													if not self.legacy:
  																			a_					=		SPIECE_UNDERLINE + text.replace(_UpperCAmelCase			,    """ """		)
  													return super().tokenize(_UpperCAmelCase			,    **_UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													if not self.legacy:
  																			a_					=		text.startswith(_UpperCAmelCase		)
  																			if is_first:
  																									a_					=		text[1:]
  
  													a_					=		self.sp_model.encode(_UpperCAmelCase			,    out_type=_UpperCAmelCase		)
  
  													if not self.legacy and not is_first and not text.startswith(""" """		) and tokens[0].startswith(_UpperCAmelCase		):
  																			a_					=		([tokens[0][1:]] if len(tokens[0]		) > 1 else []) + tokens[1:]
  													return tokens
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													return self.sp_model.piece_to_id(_UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		self.sp_model.IdToPiece(_UpperCAmelCase		)
  													return token
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		[]
  													a_					=		""""""
  													a_					=		False
  													for i, token in enumerate(_UpperCAmelCase		):
  																			# make sure that special tokens are not decoded using sentencepiece model
  																			if token in self.all_special_tokens:
  																									if not prev_is_special and i != 0:
  																															out_string += " "
  																									out_string += self.sp_model.decode(_UpperCAmelCase		) + token
  																									a_					=		True
  																									a_					=		[]
  																			else:
  																									current_sub_tokens.append(_UpperCAmelCase		)
  																									a_					=		False
  													out_string += self.sp_model.decode(_UpperCAmelCase		)
  													return out_string
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase = None		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													if not os.path.isdir(_UpperCAmelCase		):
  																			logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
  																			return
  													a_					=		os.path.join(
  													    _UpperCAmelCase			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
  
  													if os.path.abspath(self.vocab_file		) != os.path.abspath(_UpperCAmelCase		) and os.path.isfile(self.vocab_file		):
  																			copyfile(self.vocab_file			,    _UpperCAmelCase		)
  													elif not os.path.isfile(self.vocab_file		):
  																			with open(_UpperCAmelCase			,    """wb"""		) as fi:
  																									a_					=		self.sp_model.serialized_model_proto()
  																									fi.write(_UpperCAmelCase		)
  
  													return (out_vocab_file,)
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase=None		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		[self.bos_token_id] if self.add_bos_token else []
  													a_					=		[self.eos_token_id] if self.add_eos_token else []
  
  													a_					=		bos_token_id + token_ids_a + eos_token_id
  
  													if token_ids_a is not None:
  																			a_					=		output + bos_token_id + token_ids_a + eos_token_id
  
  													return output
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase = None			,    _UpperCAmelCase = False		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													if already_has_special_tokens:
  																			return super().get_special_tokens_mask(
  																			    token_ids_a=_UpperCAmelCase			,    token_ids_a=_UpperCAmelCase			,    already_has_special_tokens=_UpperCAmelCase		)
  
  													a_					=		[1] if self.add_bos_token else []
  													a_					=		[1] if self.add_eos_token else []
  
  													if token_ids_a is None:
  																			return bos_token_id + ([0] * len(_UpperCAmelCase		)) + eos_token_id
  													return (
  													    bos_token_id
  													    + ([0] * len(_UpperCAmelCase		))
  													    + eos_token_id
  													    + bos_token_id
  													    + ([0] * len(_UpperCAmelCase		))
  													    + eos_token_id
  													)
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase = None		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		[self.bos_token_id] if self.add_bos_token else []
  													a_					=		[self.eos_token_id] if self.add_eos_token else []
  
  													a_					=		[0] * len(bos_token_id + token_ids_a + eos_token_id		)
  
  													if token_ids_a is not None:
  																			output += [1] * len(bos_token_id + token_ids_a + eos_token_id		)
  
  													return output
  
  
  
  
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		list(conversation.iter_texts()		)
  													if not all([is_user for is_user, msg in dialogue[::2]]		) or not all(
  													    [not is_user for is_user, msg in dialogue[1::2]]		):
  																			raise ValueError(
  																			    """The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)"""		)
  
  													a_					=		[]
  													if len(conversation.past_user_inputs		) > 0:
  																			if not conversation.past_user_inputs[0].startswith(_UpperCAmelCase		) or E_SYS not in conversation.past_user_inputs[0]:
  																									a_					=		(
  																									    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
  																									)
  													elif not dialogue[0][1].startswith(_UpperCAmelCase		) or E_SYS not in dialogue[0][1]:
  																			a_					=		(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])
  
  													dialog_tokens += sum(
  													    [
  													        [self.bos_token_id]
  													        + self.encode(
  													            f"{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} "			,    add_special_tokens=_UpperCAmelCase		)
  													        + [self.eos_token_id]
  													        for prompt, answer in zip(dialogue[::2]			,    dialogue[1::2]		)
  													    ]			,    []			,    )
  													if not (dialogue[-1][0]):
  																			raise ValueError(f"Last message must be from user, got {dialogue[-1]['role']}"		)
  													dialog_tokens += [self.bos_token_id] + self.encode(
  													    f"{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"			,    add_special_tokens=_UpperCAmelCase		)
  													return dialog_tokens
style_context: 
  
  
  import unittest
  
  import numpy as np
  import timeout_decorator  # noqa
  
  from transformers import MBartConfig, is_flax_available
  from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow
  from transformers.utils import cached_property
  
  from ...generation.test_flax_utils import FlaxGenerationTesterMixin
  from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
  
  
  if is_flax_available():
  	import os
  
  	# The slow tests are often failing with OOM error on GPU
  	# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
  	# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
  	A_    :					List[Any]             ="""platform"""
  
  	import jax
  	import jax.numpy as jnp
  
  	from transformers import AutoTokenizer
  	from transformers.models.mbart.modeling_flax_mbart import (
  	    FlaxMBartForConditionalGeneration,
  	    FlaxMBartForQuestionAnswering,
  	    FlaxMBartForSequenceClassification,
  	    FlaxMBartModel,
  	    shift_tokens_right,
  	)
  
  
  
  
  def   lowerCamelCase_       (					UpperCAmelCase__       ,    UpperCAmelCase__       ,    UpperCAmelCase__=None       ,    UpperCAmelCase__=None       ,    UpperCAmelCase__=None       ,    UpperCAmelCase__=None       ,    UpperCAmelCase__=None       ,    UpperCAmelCase__=None       ,    ):
  
  
  
  						"""simple docstring"""
  
  
  
  
  						if attention_mask is None:
  												a_					=		np.where(input_ids != config.pad_token_id       ,    1       ,    0      )
  						if decoder_attention_mask is None:
  												a_					=		np.where(decoder_input_ids != config.pad_token_id       ,    1       ,    0      )
  						if head_mask is None:
  												a_					=		np.ones((config.encoder_layers, config.encoder_attention_heads)      )
  						if decoder_head_mask is None:
  												a_					=		np.ones((config.decoder_layers, config.decoder_attention_heads)      )
  						if cross_attn_head_mask is None:
  												a_					=		np.ones((config.decoder_layers, config.decoder_attention_heads)      )
  						return {
  						    "input_ids": input_ids,
  						    "decoder_input_ids": decoder_input_ids,
  						    "attention_mask": attention_mask,
  						    "decoder_attention_mask": decoder_attention_mask,
  						}
  
  
  
  
  
  class        lowercase_				( unittest.TestCase):
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							def __init__(   self			,    _UpperCAmelCase			,    _UpperCAmelCase=13			,    _UpperCAmelCase=7			,    _UpperCAmelCase=True			,    _UpperCAmelCase=False			,    _UpperCAmelCase=99			,    _UpperCAmelCase=16			,    _UpperCAmelCase=2			,    _UpperCAmelCase=4			,    _UpperCAmelCase=4			,    _UpperCAmelCase="gelu"			,    _UpperCAmelCase=0.1			,    _UpperCAmelCase=0.1			,    _UpperCAmelCase=32			,    _UpperCAmelCase=2			,    _UpperCAmelCase=1			,    _UpperCAmelCase=0			,    _UpperCAmelCase=2			,    _UpperCAmelCase=0.0_2			,    ):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		parent
  													a_					=		batch_size
  													a_					=		seq_length
  													a_					=		is_training
  													a_					=		use_labels
  													a_					=		vocab_size
  													a_					=		hidden_size
  													a_					=		num_hidden_layers
  													a_					=		num_attention_heads
  													a_					=		intermediate_size
  													a_					=		hidden_act
  													a_					=		hidden_dropout_prob
  													a_					=		attention_probs_dropout_prob
  													a_					=		max_position_embeddings
  													a_					=		eos_token_id
  													a_					=		pad_token_id
  													a_					=		bos_token_id
  													a_					=		decoder_start_token_id
  													a_					=		initializer_range
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		np.clip(ids_tensor([self.batch_size, self.seq_length - 1]			,    self.vocab_size		)			,    3			,    self.vocab_size		)
  													a_					=		np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)			,    dtype=np.intaa		))			,    -1		)
  
  													a_					=		shift_tokens_right(_UpperCAmelCase			,    1		)
  
  													a_					=		MBartConfig(
  													    vocab_size=self.vocab_size			,    d_model=self.hidden_size			,    encoder_layers=self.num_hidden_layers			,    decoder_layers=self.num_hidden_layers			,    encoder_attention_heads=self.num_attention_heads			,    decoder_attention_heads=self.num_attention_heads			,    encoder_ffn_dim=self.intermediate_size			,    decoder_ffn_dim=self.intermediate_size			,    dropout=self.hidden_dropout_prob			,    attention_dropout=self.attention_probs_dropout_prob			,    max_position_embeddings=self.max_position_embeddings			,    eos_token_id=self.eos_token_id			,    bos_token_id=self.bos_token_id			,    pad_token_id=self.pad_token_id			,    decoder_start_token_id=self.decoder_start_token_id			,    initializer_range=self.initializer_range			,    use_cache=_UpperCAmelCase			,    )
  													a_					=		prepare_mbart_inputs_dict(_UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase		)
  													return config, inputs_dict
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_					=		self.prepare_config_and_inputs()
  													return config, inputs_dict
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		20
  													a_					=		model_class_name(_UpperCAmelCase		)
  
  													a_					=		model.encode(inputs_dict["""input_ids"""]		)
  
  													a_			,   a_					=		(
  													    inputs_dict["""decoder_input_ids"""],
  													    inputs_dict["""decoder_attention_mask"""],
  													)
  
  													a_					=		model.init_cache(decoder_input_ids.shape[0]			,    _UpperCAmelCase			,    _UpperCAmelCase		)
  													a_					=		jnp.ones((decoder_input_ids.shape[0], max_decoder_length)			,    dtype="""i4"""		)
  
  													a_					=		jnp.broadcast_to(
  													    jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]			,    (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)			,    )
  													a_					=		model.decode(
  													    decoder_input_ids[:, :-1]			,    _UpperCAmelCase			,    decoder_attention_mask=_UpperCAmelCase			,    past_key_values=_UpperCAmelCase			,    decoder_position_ids=_UpperCAmelCase			,    )
  
  													a_					=		jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]			,    dtype="""i4"""		)
  													a_					=		model.decode(
  													    decoder_input_ids[:, -1:]			,    _UpperCAmelCase			,    decoder_attention_mask=_UpperCAmelCase			,    past_key_values=outputs_cache.past_key_values			,    decoder_position_ids=_UpperCAmelCase			,    )
  
  													a_					=		model.decode(_UpperCAmelCase			,    _UpperCAmelCase		)
  
  													a_					=		np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
  													self.parent.assertTrue(diff < 1e-3			,    msg=f"Max diff is {diff}"		)
  
  
  
  
  
  
  							def 						lowercase__     (   self			,    _UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		20
  													a_					=		model_class_name(_UpperCAmelCase		)
  
  													a_					=		model.encode(inputs_dict["""input_ids"""]		)
  
  													a_			,   a_					=		(
  													    inputs_dict["""decoder_input_ids"""],
  													    inputs_dict["""decoder_attention_mask"""],
  													)
  
  													a_					=		jnp.concatenate(
  													    [
  													        decoder_attention_mask,
  													        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])		),
  													    ]			,    axis=-1			,    )
  
  													a_					=		model.init_cache(decoder_input_ids.shape[0]			,    _UpperCAmelCase			,    _UpperCAmelCase		)
  													a_					=		jnp.broadcast_to(
  													    jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]			,    (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)			,    )
  
  													a_					=		model.decode(
  													    decoder_input_ids[:, :-1]			,    _UpperCAmelCase			,    decoder_attention_mask=_UpperCAmelCase			,    past_key_values=_UpperCAmelCase			,    decoder_position_ids=_UpperCAmelCase			,    )
  													a_					=		jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]			,    dtype="""i4"""		)
  													a_					=		model.decode(
  													    decoder_input_ids[:, -1:]			,    _UpperCAmelCase			,    past_key_values=outputs_cache.past_key_values			,    decoder_attention_mask=_UpperCAmelCase			,    decoder_position_ids=_UpperCAmelCase			,    )
  
  													a_					=		model.decode(_UpperCAmelCase			,    _UpperCAmelCase			,    decoder_attention_mask=_UpperCAmelCase		)
  
  													a_					=		np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
  													self.parent.assertTrue(diff < 1e-3			,    msg=f"Max diff is {diff}"		)
  
  
  
  
  
  @require_flax
  class        lowercase_				( unittest.TestCase):
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							snake_case_        =		99
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		np.array(
  													    [
  													        [71, 82, 18, 33, 46, 91, 2],
  													        [68, 34, 26, 58, 30, 82, 2],
  													        [5, 97, 17, 39, 94, 40, 2],
  													        [76, 83, 94, 25, 70, 78, 2],
  													        [87, 59, 41, 35, 48, 66, 2],
  													        [55, 13, 16, 58, 5, 2, 1],  # note padding
  													        [64, 27, 31, 51, 12, 75, 2],
  													        [52, 64, 86, 17, 83, 39, 2],
  													        [48, 61, 9, 24, 71, 82, 2],
  													        [26, 1, 60, 48, 22, 13, 2],
  													        [21, 5, 62, 28, 14, 76, 2],
  													        [45, 98, 37, 86, 59, 48, 2],
  													        [70, 70, 50, 9, 28, 0, 2],
  													    ]			,    dtype=np.intaa			,    )
  
  													a_					=		input_ids.shape[0]
  													a_					=		MBartConfig(
  													    vocab_size=self.vocab_size			,    d_model=24			,    encoder_layers=2			,    decoder_layers=2			,    encoder_attention_heads=2			,    decoder_attention_heads=2			,    encoder_ffn_dim=32			,    decoder_ffn_dim=32			,    max_position_embeddings=48			,    eos_token_id=2			,    pad_token_id=1			,    bos_token_id=0			,    )
  													return config, input_ids, batch_size
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_			,   a_					=		self._get_config_and_data()
  													a_					=		FlaxMBartForSequenceClassification(_UpperCAmelCase		)
  													a_					=		model(input_ids=_UpperCAmelCase			,    decoder_input_ids=_UpperCAmelCase		)
  													a_					=		(batch_size, config.num_labels)
  													self.assertEqual(outputs["""logits"""].shape			,    _UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_			,   a_					=		self._get_config_and_data()
  													a_					=		FlaxMBartForQuestionAnswering(_UpperCAmelCase		)
  													a_					=		model(input_ids=_UpperCAmelCase		)
  
  													self.assertEqual(outputs["""start_logits"""].shape			,    input_ids.shape		)
  													self.assertEqual(outputs["""end_logits"""].shape			,    input_ids.shape		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_			,   a_					=		self._get_config_and_data()
  													a_					=		FlaxMBartForConditionalGeneration(_UpperCAmelCase		)
  													a_					=		lm_model(input_ids=_UpperCAmelCase		)
  													a_					=		(batch_size, input_ids.shape[1], config.vocab_size)
  													self.assertEqual(outputs["""logits"""].shape			,    _UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		MBartConfig(
  													    vocab_size=self.vocab_size			,    d_model=14			,    encoder_layers=2			,    decoder_layers=2			,    encoder_attention_heads=2			,    decoder_attention_heads=2			,    encoder_ffn_dim=8			,    decoder_ffn_dim=8			,    max_position_embeddings=48			,    )
  													a_					=		FlaxMBartForConditionalGeneration(_UpperCAmelCase		)
  													a_					=		np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]			,    dtype=np.intaa		)
  													a_					=		np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]			,    dtype=np.intaa		)
  													a_					=		lm_model(input_ids=_UpperCAmelCase			,    decoder_input_ids=_UpperCAmelCase		)
  													a_					=		(*summary.shape, config.vocab_size)
  													self.assertEqual(outputs["""logits"""].shape			,    _UpperCAmelCase		)
  
  
  
  
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]			,    dtype=np.intaa		)
  													a_					=		shift_tokens_right(_UpperCAmelCase			,    1		)
  													a_					=		np.equal(_UpperCAmelCase			,    1		).astype(np.floataa		).sum()
  													a_					=		np.equal(_UpperCAmelCase			,    1		).astype(np.floataa		).sum()
  													self.assertEqual(shifted.shape			,    input_ids.shape		)
  													self.assertEqual(_UpperCAmelCase			,    n_pad_before - 1		)
  													self.assertTrue(np.equal(shifted[:, 0]			,    2		).all()		)
  
  
  
  
  
  @require_flax
  class        lowercase_				( UpperCamelCase__			,unittest.TestCase			,UpperCamelCase__):
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							snake_case_        =		True
  							snake_case_        =		(
  							    (
  							        FlaxMBartModel,
  							        FlaxMBartForConditionalGeneration,
  							        FlaxMBartForSequenceClassification,
  							        FlaxMBartForQuestionAnswering,
  							    )
  							    if is_flax_available()
  							    else ()
  							)
  							snake_case_        =		(FlaxMBartForConditionalGeneration,) if is_flax_available() else ()
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		FlaxMBartModelTester(self		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_					=		self.model_tester.prepare_config_and_inputs()
  													for model_class in self.all_model_classes:
  																			self.model_tester.check_use_cache_forward(_UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_					=		self.model_tester.prepare_config_and_inputs()
  													for model_class in self.all_model_classes:
  																			self.model_tester.check_use_cache_forward_with_attn_mask(_UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_					=		self.model_tester.prepare_config_and_inputs_for_common()
  
  													for model_class in self.all_model_classes:
  																			with self.subTest(model_class.__name__		):
  																									a_					=		self._prepare_for_class(_UpperCAmelCase			,    _UpperCAmelCase		)
  																									a_					=		model_class(_UpperCAmelCase		)
  
  																									@jax.jit
  																									def encode_jitted(_UpperCAmelCase			,    _UpperCAmelCase=None			,    **_UpperCAmelCase		):
  																															return model.encode(input_ids=_UpperCAmelCase			,    attention_mask=_UpperCAmelCase		)
  
  																									with self.subTest("""JIT Enabled"""		):
  																															a_					=		encode_jitted(**_UpperCAmelCase		).to_tuple()
  
  																									with self.subTest("""JIT Disabled"""		):
  																															with jax.disable_jit():
  																																					a_					=		encode_jitted(**_UpperCAmelCase		).to_tuple()
  
  																									self.assertEqual(len(_UpperCAmelCase		)			,    len(_UpperCAmelCase		)		)
  																									for jitted_output, output in zip(_UpperCAmelCase			,    _UpperCAmelCase		):
  																															self.assertEqual(jitted_output.shape			,    output.shape		)
  
  
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_			,   a_					=		self.model_tester.prepare_config_and_inputs_for_common()
  
  													for model_class in self.all_model_classes:
  																			with self.subTest(model_class.__name__		):
  																									a_					=		model_class(_UpperCAmelCase		)
  																									a_					=		model.encode(inputs_dict["""input_ids"""]			,    inputs_dict["""attention_mask"""]		)
  
  																									a_					=		{
  																									    """decoder_input_ids""": inputs_dict["""decoder_input_ids"""],
  																									    """decoder_attention_mask""": inputs_dict["""decoder_attention_mask"""],
  																									    """encoder_outputs""": encoder_outputs,
  																									}
  
  																									@jax.jit
  																									def decode_jitted(_UpperCAmelCase			,    _UpperCAmelCase			,    _UpperCAmelCase		):
  																															return model.decode(
  																															    decoder_input_ids=_UpperCAmelCase			,    decoder_attention_mask=_UpperCAmelCase			,    encoder_outputs=_UpperCAmelCase			,    )
  
  																									with self.subTest("""JIT Enabled"""		):
  																															a_					=		decode_jitted(**_UpperCAmelCase		).to_tuple()
  
  																									with self.subTest("""JIT Disabled"""		):
  																															with jax.disable_jit():
  																																					a_					=		decode_jitted(**_UpperCAmelCase		).to_tuple()
  
  																									self.assertEqual(len(_UpperCAmelCase		)			,    len(_UpperCAmelCase		)		)
  																									for jitted_output, output in zip(_UpperCAmelCase			,    _UpperCAmelCase		):
  																															self.assertEqual(jitted_output.shape			,    output.shape		)
  
  
  
  
  
  
  							@slow
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													for model_class_name in self.all_model_classes:
  																			a_					=		model_class_name.from_pretrained("""facebook/mbart-large-cc25"""			,    from_pt=_UpperCAmelCase		)
  																			# FlaxMBartForSequenceClassification expects eos token in input_ids
  																			a_					=		np.ones((1, 1)		) * model.config.eos_token_id
  																			a_					=		model(_UpperCAmelCase		)
  																			self.assertIsNotNone(_UpperCAmelCase		)
  
  
  
  
  
  @require_flax
  @require_sentencepiece
  @require_tokenizers
  class        lowercase_				( unittest.TestCase):
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							snake_case_        =		[
  							    ''' UN Chief Says There Is No Military Solution in Syria''',
  							]
  							snake_case_        =		[
  							    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
  							]
  							snake_case_        =		'''facebook/mbart-large-en-ro'''
  
  
  							@cached_property
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													return AutoTokenizer.from_pretrained(self.model_name		)
  
  
  							@cached_property
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		FlaxMBartForConditionalGeneration.from_pretrained(self.model_name			,    from_pt=_UpperCAmelCase		)
  													return model
  
  
  							def 						lowercase__     (   self			,    **_UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		self.translate_src_text(**_UpperCAmelCase		)
  													self.assertListEqual(self.expected_text			,    _UpperCAmelCase		)
  
  
  							def 						lowercase__     (   self			,    **_UpperCAmelCase		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													a_					=		self.tokenizer(self.src_text			,    **_UpperCAmelCase			,    return_tensors="""np"""		)
  													a_					=		self.model.generate(
  													    model_inputs.input_ids			,    attention_mask=model_inputs.attention_mask			,    decoder_start_token_id=self.tokenizer.lang_code_to_id["""ro_RO"""]			,    early_stopping=_UpperCAmelCase			,    num_beams=2			,    ).sequences
  													a_					=		self.tokenizer.batch_decode(_UpperCAmelCase			,    skip_special_tokens=_UpperCAmelCase		)
  													return generated_words
  
  
  
  
  
  
  							@slow
  							def 						lowercase__     (   self		):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													self._assert_generated_batch_equal_expected()