code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import unittest
  
  import numpy as np
  import timeout_decorator  # noqa
  
  from transformers import MBartConfig, is_flax_available
  from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow
  from transformers.utils import cached_property
  
  from ...generation.test_flax_utils import FlaxGenerationTesterMixin
  from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
  
  
  if is_flax_available():
  						import os
  
  						# The slow tests are often failing with OOM error on GPU
  						# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
  						# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
  						_lowerCamelCase							:		Tuple										=						'platform'
  
  						import jax
  						import jax.numpy as jnp
  
  						from transformers import AutoTokenizer
  						from transformers.models.mbart.modeling_flax_mbart import (
  						    FlaxMBartForConditionalGeneration,
  						    FlaxMBartForQuestionAnswering,
  						    FlaxMBartForSequenceClassification,
  						    FlaxMBartModel,
  						    shift_tokens_right,
  						)
  
  
  
  
  
  
  def 	_lowercase	(			lowercase_		,							lowercase_		,							lowercase_=None		,							lowercase_=None		,							lowercase_=None		,							lowercase_=None		,							lowercase_=None		,							lowercase_=None		,							):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if attention_mask is None:
  				_snake_case			 = np.where(input_ids != config.pad_token_id		,							1		,							0						)
  		if decoder_attention_mask is None:
  				_snake_case			 = np.where(decoder_input_ids != config.pad_token_id		,							1		,							0						)
  		if head_mask is None:
  				_snake_case			 = np.ones((config.encoder_layers, config.encoder_attention_heads)						)
  		if decoder_head_mask is None:
  				_snake_case			 = np.ones((config.decoder_layers, config.decoder_attention_heads)						)
  		if cross_attn_head_mask is None:
  				_snake_case			 = np.ones((config.decoder_layers, config.decoder_attention_heads)						)
  		return {
  		    "input_ids": input_ids,
  		    "decoder_input_ids": decoder_input_ids,
  		    "attention_mask": attention_mask,
  		    "decoder_attention_mask": decoder_attention_mask,
  		}
  
  
  
  
  
  class   lowerCamelCase_							( unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					def __init__(					self,						__UpperCamelCase,						__UpperCamelCase=13,						__UpperCamelCase=7,						__UpperCamelCase=True,						__UpperCamelCase=False,						__UpperCamelCase=99,						__UpperCamelCase=16,						__UpperCamelCase=2,						__UpperCamelCase=4,						__UpperCamelCase=4,						__UpperCamelCase="gelu",						__UpperCamelCase=0.1,						__UpperCamelCase=0.1,						__UpperCamelCase=32,						__UpperCamelCase=2,						__UpperCamelCase=1,						__UpperCamelCase=0,						__UpperCamelCase=2,						__UpperCamelCase=0.02,						):
  							_snake_case			 = parent
  							_snake_case			 = batch_size
  							_snake_case			 = seq_length
  							_snake_case			 = is_training
  							_snake_case			 = use_labels
  							_snake_case			 = vocab_size
  							_snake_case			 = hidden_size
  							_snake_case			 = num_hidden_layers
  							_snake_case			 = num_attention_heads
  							_snake_case			 = intermediate_size
  							_snake_case			 = hidden_act
  							_snake_case			 = hidden_dropout_prob
  							_snake_case			 = attention_probs_dropout_prob
  							_snake_case			 = max_position_embeddings
  							_snake_case			 = eos_token_id
  							_snake_case			 = pad_token_id
  							_snake_case			 = bos_token_id
  							_snake_case			 = decoder_start_token_id
  							_snake_case			 = initializer_range
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = np.clip(ids_tensor([self.batch_size, self.seq_length - 1],						self.vocab_size				),						3,						self.vocab_size				)
  							_snake_case			 = np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1),						dtype=np.intaa				)),						-1				)
  
  							_snake_case			 = shift_tokens_right(__UpperCamelCase,						1				)
  
  							_snake_case			 = MBartConfig(
  							    vocab_size=self.vocab_size,						d_model=self.hidden_size,						encoder_layers=self.num_hidden_layers,						decoder_layers=self.num_hidden_layers,						encoder_attention_heads=self.num_attention_heads,						decoder_attention_heads=self.num_attention_heads,						encoder_ffn_dim=self.intermediate_size,						decoder_ffn_dim=self.intermediate_size,						dropout=self.hidden_dropout_prob,						attention_dropout=self.attention_probs_dropout_prob,						max_position_embeddings=self.max_position_embeddings,						eos_token_id=self.eos_token_id,						bos_token_id=self.bos_token_id,						pad_token_id=self.pad_token_id,						decoder_start_token_id=self.decoder_start_token_id,						initializer_range=self.initializer_range,						use_cache=__UpperCamelCase,						)
  							_snake_case			 = prepare_mbart_inputs_dict(__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				)
  							return config, inputs_dict
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case			 = self.prepare_config_and_inputs()
  							return config, inputs_dict
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				):
  							_snake_case			 = 20
  							_snake_case			 = model_class_name(__UpperCamelCase				)
  
  							_snake_case			 = model.encode(inputs_dict["input_ids"]				)
  
  							_snake_case ,					_snake_case			 = (
  							    inputs_dict["decoder_input_ids"],
  							    inputs_dict["decoder_attention_mask"],
  							)
  
  							_snake_case			 = model.init_cache(decoder_input_ids.shape[0],						__UpperCamelCase,						__UpperCamelCase				)
  							_snake_case			 = jnp.ones((decoder_input_ids.shape[0], max_decoder_length),						dtype="i4"				)
  
  							_snake_case			 = jnp.broadcast_to(
  							    jnp.arange(decoder_input_ids.shape[-1] - 1				)[None, :],						(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1),						)
  							_snake_case			 = model.decode(
  							    decoder_input_ids[:, :-1],						__UpperCamelCase,						decoder_attention_mask=__UpperCamelCase,						past_key_values=__UpperCamelCase,						decoder_position_ids=__UpperCamelCase,						)
  
  							_snake_case			 = jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]],						dtype="i4"				)
  							_snake_case			 = model.decode(
  							    decoder_input_ids[:, -1:],						__UpperCamelCase,						decoder_attention_mask=__UpperCamelCase,						past_key_values=outputs_cache.past_key_values,						decoder_position_ids=__UpperCamelCase,						)
  
  							_snake_case			 = model.decode(__UpperCamelCase,						__UpperCamelCase				)
  
  							_snake_case			 = np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])				)				)
  							self.parent.assertTrue(diff < 1E-3,						msg=f'''Max diff is {diff}'''				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				):
  							_snake_case			 = 20
  							_snake_case			 = model_class_name(__UpperCamelCase				)
  
  							_snake_case			 = model.encode(inputs_dict["input_ids"]				)
  
  							_snake_case ,					_snake_case			 = (
  							    inputs_dict["decoder_input_ids"],
  							    inputs_dict["decoder_attention_mask"],
  							)
  
  							_snake_case			 = jnp.concatenate(
  							    [
  							        decoder_attention_mask,
  							        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])				),
  							    ],						axis=-1,						)
  
  							_snake_case			 = model.init_cache(decoder_input_ids.shape[0],						__UpperCamelCase,						__UpperCamelCase				)
  							_snake_case			 = jnp.broadcast_to(
  							    jnp.arange(decoder_input_ids.shape[-1] - 1				)[None, :],						(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1),						)
  
  							_snake_case			 = model.decode(
  							    decoder_input_ids[:, :-1],						__UpperCamelCase,						decoder_attention_mask=__UpperCamelCase,						past_key_values=__UpperCamelCase,						decoder_position_ids=__UpperCamelCase,						)
  							_snake_case			 = jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]],						dtype="i4"				)
  							_snake_case			 = model.decode(
  							    decoder_input_ids[:, -1:],						__UpperCamelCase,						past_key_values=outputs_cache.past_key_values,						decoder_attention_mask=__UpperCamelCase,						decoder_position_ids=__UpperCamelCase,						)
  
  							_snake_case			 = model.decode(__UpperCamelCase,						__UpperCamelCase,						decoder_attention_mask=__UpperCamelCase				)
  
  							_snake_case			 = np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])				)				)
  							self.parent.assertTrue(diff < 1E-3,						msg=f'''Max diff is {diff}'''				)
  
  
  
  
  
  @require_flax
  class   lowerCamelCase_							( unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				9_9
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = np.array(
  							    [
  							        [71, 82, 18, 33, 46, 91, 2],
  							        [68, 34, 26, 58, 30, 82, 2],
  							        [5, 97, 17, 39, 94, 40, 2],
  							        [76, 83, 94, 25, 70, 78, 2],
  							        [87, 59, 41, 35, 48, 66, 2],
  							        [55, 13, 16, 58, 5, 2, 1],  # note padding
  							        [64, 27, 31, 51, 12, 75, 2],
  							        [52, 64, 86, 17, 83, 39, 2],
  							        [48, 61, 9, 24, 71, 82, 2],
  							        [26, 1, 60, 48, 22, 13, 2],
  							        [21, 5, 62, 28, 14, 76, 2],
  							        [45, 98, 37, 86, 59, 48, 2],
  							        [70, 70, 50, 9, 28, 0, 2],
  							    ],						dtype=np.intaa,						)
  
  							_snake_case			 = input_ids.shape[0]
  							_snake_case			 = MBartConfig(
  							    vocab_size=self.vocab_size,						d_model=24,						encoder_layers=2,						decoder_layers=2,						encoder_attention_heads=2,						decoder_attention_heads=2,						encoder_ffn_dim=32,						decoder_ffn_dim=32,						max_position_embeddings=48,						eos_token_id=2,						pad_token_id=1,						bos_token_id=0,						)
  							return config, input_ids, batch_size
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case ,					_snake_case			 = self._get_config_and_data()
  							_snake_case			 = FlaxMBartForSequenceClassification(__UpperCamelCase				)
  							_snake_case			 = model(input_ids=__UpperCamelCase,						decoder_input_ids=__UpperCamelCase				)
  							_snake_case			 = (batch_size, config.num_labels)
  							self.assertEqual(outputs["logits"].shape,						__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case ,					_snake_case			 = self._get_config_and_data()
  							_snake_case			 = FlaxMBartForQuestionAnswering(__UpperCamelCase				)
  							_snake_case			 = model(input_ids=__UpperCamelCase				)
  
  							self.assertEqual(outputs["start_logits"].shape,						input_ids.shape				)
  							self.assertEqual(outputs["end_logits"].shape,						input_ids.shape				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case ,					_snake_case			 = self._get_config_and_data()
  							_snake_case			 = FlaxMBartForConditionalGeneration(__UpperCamelCase				)
  							_snake_case			 = lm_model(input_ids=__UpperCamelCase				)
  							_snake_case			 = (batch_size, input_ids.shape[1], config.vocab_size)
  							self.assertEqual(outputs["logits"].shape,						__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = MBartConfig(
  							    vocab_size=self.vocab_size,						d_model=14,						encoder_layers=2,						decoder_layers=2,						encoder_attention_heads=2,						decoder_attention_heads=2,						encoder_ffn_dim=8,						decoder_ffn_dim=8,						max_position_embeddings=48,						)
  							_snake_case			 = FlaxMBartForConditionalGeneration(__UpperCamelCase				)
  							_snake_case			 = np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]],						dtype=np.intaa				)
  							_snake_case			 = np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]],						dtype=np.intaa				)
  							_snake_case			 = lm_model(input_ids=__UpperCamelCase,						decoder_input_ids=__UpperCamelCase				)
  							_snake_case			 = (*summary.shape, config.vocab_size)
  							self.assertEqual(outputs["logits"].shape,						__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]],						dtype=np.intaa				)
  							_snake_case			 = shift_tokens_right(__UpperCamelCase,						1				)
  							_snake_case			 = np.equal(__UpperCamelCase,						1				).astype(np.floataa				).sum()
  							_snake_case			 = np.equal(__UpperCamelCase,						1				).astype(np.floataa				).sum()
  							self.assertEqual(shifted.shape,						input_ids.shape				)
  							self.assertEqual(__UpperCamelCase,						n_pad_before - 1				)
  							self.assertTrue(np.equal(shifted[:, 0],						2				).all()				)
  
  
  
  
  
  @require_flax
  class   lowerCamelCase_							( UpperCAmelCase_, unittest.TestCase, UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				True
  					lowercase__					=				(
  					    (
  					        FlaxMBartModel,
  					        FlaxMBartForConditionalGeneration,
  					        FlaxMBartForSequenceClassification,
  					        FlaxMBartForQuestionAnswering,
  					    )
  					    if is_flax_available()
  					    else ()
  					)
  					lowercase__					=				(FlaxMBartForConditionalGeneration,) if is_flax_available() else ()
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = FlaxMBartModelTester(self				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case			 = self.model_tester.prepare_config_and_inputs()
  							for model_class in self.all_model_classes:
  									self.model_tester.check_use_cache_forward(__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case			 = self.model_tester.prepare_config_and_inputs()
  							for model_class in self.all_model_classes:
  									self.model_tester.check_use_cache_forward_with_attn_mask(__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case			 = self.model_tester.prepare_config_and_inputs_for_common()
  
  							for model_class in self.all_model_classes:
  									with self.subTest(model_class.__name__				):
  											_snake_case			 = self._prepare_for_class(__UpperCamelCase,						__UpperCamelCase				)
  											_snake_case			 = model_class(__UpperCamelCase				)
  
  											@jax.jit
  											def encode_jitted(__UpperCamelCase,						__UpperCamelCase=None,						**__UpperCamelCase				):
  													return model.encode(input_ids=__UpperCamelCase,						attention_mask=__UpperCamelCase				)
  
  											with self.subTest("JIT Enabled"				):
  													_snake_case			 = encode_jitted(**__UpperCamelCase				).to_tuple()
  
  											with self.subTest("JIT Disabled"				):
  													with jax.disable_jit():
  															_snake_case			 = encode_jitted(**__UpperCamelCase				).to_tuple()
  
  											self.assertEqual(len(__UpperCamelCase				),						len(__UpperCamelCase				)				)
  											for jitted_output, output in zip(__UpperCamelCase,						__UpperCamelCase				):
  													self.assertEqual(jitted_output.shape,						output.shape				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case ,					_snake_case			 = self.model_tester.prepare_config_and_inputs_for_common()
  
  							for model_class in self.all_model_classes:
  									with self.subTest(model_class.__name__				):
  											_snake_case			 = model_class(__UpperCamelCase				)
  											_snake_case			 = model.encode(inputs_dict["input_ids"],						inputs_dict["attention_mask"]				)
  
  											_snake_case			 = {
  											    "decoder_input_ids": inputs_dict["decoder_input_ids"],
  											    "decoder_attention_mask": inputs_dict["decoder_attention_mask"],
  											    "encoder_outputs": encoder_outputs,
  											}
  
  											@jax.jit
  											def decode_jitted(__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				):
  													return model.decode(
  													    decoder_input_ids=__UpperCamelCase,						decoder_attention_mask=__UpperCamelCase,						encoder_outputs=__UpperCamelCase,						)
  
  											with self.subTest("JIT Enabled"				):
  													_snake_case			 = decode_jitted(**__UpperCamelCase				).to_tuple()
  
  											with self.subTest("JIT Disabled"				):
  													with jax.disable_jit():
  															_snake_case			 = decode_jitted(**__UpperCamelCase				).to_tuple()
  
  											self.assertEqual(len(__UpperCamelCase				),						len(__UpperCamelCase				)				)
  											for jitted_output, output in zip(__UpperCamelCase,						__UpperCamelCase				):
  													self.assertEqual(jitted_output.shape,						output.shape				)
  
  
  
  
  
  
  					@slow
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							for model_class_name in self.all_model_classes:
  									_snake_case			 = model_class_name.from_pretrained("facebook/mbart-large-cc25",						from_pt=__UpperCamelCase				)
  									# FlaxMBartForSequenceClassification expects eos token in input_ids
  									_snake_case			 = np.ones((1, 1)				) * model.config.eos_token_id
  									_snake_case			 = model(__UpperCamelCase				)
  									self.assertIsNotNone(__UpperCamelCase				)
  
  
  
  
  
  @require_flax
  @require_sentencepiece
  @require_tokenizers
  class   lowerCamelCase_							( unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				[
  					    """ UN Chief Says There Is No Military Solution in Syria""",
  					]
  					lowercase__					=				[
  					    """Şeful ONU declară că nu există o soluţie militară în Siria""",
  					]
  					lowercase__					=				"""facebook/mbart-large-en-ro"""
  
  
  
  
  
  
  					@cached_property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							return AutoTokenizer.from_pretrained(self.model_name				)
  
  
  
  
  
  
  					@cached_property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = FlaxMBartForConditionalGeneration.from_pretrained(self.model_name,						from_pt=__UpperCamelCase				)
  							return model
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						**__UpperCamelCase				):
  							_snake_case			 = self.translate_src_text(**__UpperCamelCase				)
  							self.assertListEqual(self.expected_text,						__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						**__UpperCamelCase				):
  							_snake_case			 = self.tokenizer(self.src_text,						**__UpperCamelCase,						return_tensors="np"				)
  							_snake_case			 = self.model.generate(
  							    model_inputs.input_ids,						attention_mask=model_inputs.attention_mask,						decoder_start_token_id=self.tokenizer.lang_code_to_id["ro_RO"],						early_stopping=__UpperCamelCase,						num_beams=2,						).sequences
  							_snake_case			 = self.tokenizer.batch_decode(__UpperCamelCase,						skip_special_tokens=__UpperCamelCase				)
  							return generated_words
  
  
  
  
  
  
  					@slow
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							self._assert_generated_batch_equal_expected()
  
  
  
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class   lowerCamelCase_							( UpperCAmelCase_, unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				GPTNeoXJapaneseTokenizer
  					lowercase__					=				False
  					lowercase__					=				{"""do_clean_text""": False, """add_prefix_space""": False}
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							super().setUp()
  
  							_snake_case			 = [
  							    "こん",
  							    "こんに",
  							    "にちは",
  							    "ばんは",
  							    "世界,㔺界",
  							    "、",
  							    "。",
  							    "<BR>",
  							    "<SP>",
  							    "<TAB>",
  							    "<URL>",
  							    "<EMAIL>",
  							    "<TEL>",
  							    "<DATE>",
  							    "<PRICE>",
  							    "<BLOCK>",
  							    "<KIGOU>",
  							    "<U2000U2BFF>",
  							    "<|emoji1|>",
  							    "<unk>",
  							    "<|startoftext|>",
  							    "<|endoftext|>",
  							]
  							_snake_case			 = {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
  							_snake_case			 = {"unk_token": "<unk>"}
  
  							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["vocab_file"]				)
  							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["emoji_file"]				)
  							with open(self.vocab_file,						"w",						encoding="utf-8"				) as vocab_writer:
  									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)
  							with open(self.emoji_file,						"w"				) as emoji_writer:
  									emoji_writer.write(json.dumps(__UpperCamelCase				)				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						**__UpperCamelCase				):
  							kwargs.update(self.special_tokens_map				)
  							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname,						**__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case			 = "こんにちは、世界。 \nこんばんは、㔺界。😀"
  							_snake_case			 = "こんにちは、世界。 \nこんばんは、世界。😀"
  							return input_text, output_text
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case ,					_snake_case			 = self.get_input_output_texts(__UpperCamelCase				)
  							_snake_case			 = tokenizer.encode(__UpperCamelCase,						add_special_tokens=__UpperCamelCase				)
  							_snake_case			 = tokenizer.decode(__UpperCamelCase,						clean_up_tokenization_spaces=__UpperCamelCase				)
  							return text, ids
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = self.get_tokenizer()
  
  							# Testing tokenization
  							_snake_case			 = "こんにちは、世界。　こんばんは、㔺界。"
  							_snake_case			 = ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
  							_snake_case			 = tokenizer.tokenize(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  							# Testing conversion to ids without special tokens
  							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
  							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  							# Testing conversion to ids with special tokens
  							_snake_case			 = tokens + [tokenizer.unk_token]
  							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
  							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  
  
  
  
  
  					@slow
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"				)
  
  							_snake_case			 = tokenizer.encode("ありがとう。",						add_special_tokens=__UpperCamelCase				)
  							_snake_case			 = tokenizer.encode("どういたしまして。",						add_special_tokens=__UpperCamelCase				)
  
  							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				)
  							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase,						__UpperCamelCase				)
  
  							assert encoded_sentence == ids_a
  							assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							# Intentionally convert some words to accommodate character fluctuations unique to Japanese
  							pass
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							# tokenizer has no padding token
  							pass
  
  
  
  
  
  