





'''simple docstring'''






import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
     import jax
     import jax.numpy as jnp

     from transformers.modeling_flax_pytorch_utils import (
         convert_pytorch_state_dict_to_flax,
         load_flax_weights_in_pytorch_model,
     )
     from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
     import torch


class 	_lowercase    :


   def __init__(			self     , _UpperCAmelCase     , _UpperCAmelCase=14     , _UpperCAmelCase=7     , _UpperCAmelCase=True     , _UpperCAmelCase=True     , _UpperCAmelCase=False     , _UpperCAmelCase=True     , _UpperCAmelCase=99     , _UpperCAmelCase=32     , _UpperCAmelCase=5     , _UpperCAmelCase=4     , _UpperCAmelCase=37     , _UpperCAmelCase="gelu"     , _UpperCAmelCase=0.1     , _UpperCAmelCase=0.1     , _UpperCAmelCase=512     , _UpperCAmelCase=0.02     , ):
       A							:							Optional[int]				=    parent
       A							:							Dict				=    batch_size
       A							:							Optional[Any]				=    seq_length
       A							:							List[Any]				=    is_training
       A							:							List[str]				=    use_input_mask
       A							:							Dict				=    use_token_type_ids
       A							:							int				=    use_labels
       A							:							List[str]				=    vocab_size
       A							:							List[str]				=    hidden_size
       A							:							Dict				=    num_hidden_layers
       A							:							str				=    num_attention_heads
       A							:							Dict				=    intermediate_size
       A							:							List[Any]				=    hidden_act
       A							:							Dict				=    hidden_dropout_prob
       A							:							str				=    attention_probs_dropout_prob
       A							:							Union[str, Any]				=    max_position_embeddings
       A							:							Dict				=    initializer_range
       A							:							List[Any]				=    None
       A							:							Optional[Any]				=    vocab_size - 1
       A							:							List[str]				=    vocab_size - 1
       A							:							str				=    vocab_size - 1


   def        snake_case					(			self       ):
       A							:							Dict				=    ids_tensor([self.batch_size, self.seq_length]     , self.vocab_size       )

       A							:							str				=    None
       if self.use_input_mask:
           A							:							List[str]				=    random_attention_mask([self.batch_size, self.seq_length]       )

       A							:							List[str]				=    GPTaConfig(
           vocab_size=self.vocab_size     , n_embd=self.hidden_size     , n_layer=self.num_hidden_layers     , n_head=self.num_attention_heads     , n_positions=self.max_position_embeddings     , use_cache=__UpperCamelCase     , bos_token_id=self.bos_token_id     , eos_token_id=self.eos_token_id     , pad_token_id=self.pad_token_id     , )

       return (config, input_ids, input_mask)


   def        snake_case					(			self       ):
       A							:							Any				=    self.prepare_config_and_inputs()
       A							:							Any				=    config_and_inputs
       A							:							Tuple				=    {"""input_ids""": input_ids, """attention_mask""": attention_mask}
       return config, inputs_dict


   def        snake_case					(			self       ):
       A							:							Optional[int]				=    self.prepare_config_and_inputs()

       A							:							Dict				=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
       A							:							Optional[Any]				=    ids_tensor([self.batch_size, self.seq_length]     , vocab_size=2       )

       return (
           config,
           input_ids,
           attention_mask,
           encoder_hidden_states,
           encoder_attention_mask,
       )


   def        snake_case					(			self     , _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase       ):
       A							:							Union[str, Any]				=    20
       A							:							Any				=    model_class_name(__UpperCamelCase       )

       A							:							Optional[int]				=    model.init_cache(input_ids.shape[0]     , __UpperCamelCase       )
       A							:							Dict				=    jnp.ones((input_ids.shape[0], max_decoder_length)     , dtype='''i4'''       )

       A							:							str				=    jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1       )[None, :]     , (input_ids.shape[0], input_ids.shape[-1] - 1)       )
       A							:							int				=    model(
           input_ids[:, :-1]     , attention_mask=__UpperCamelCase     , past_key_values=__UpperCamelCase     , position_ids=__UpperCamelCase     , )

       A							:							Dict				=    jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]     , dtype='''i4'''       )
       A							:							Any				=    model(
           input_ids[:, -1:]     , attention_mask=__UpperCamelCase     , past_key_values=outputs_cache.past_key_values     , position_ids=__UpperCamelCase     , )

       A							:							Union[str, Any]				=    model(__UpperCamelCase       )

       A							:							Any				=    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
       self.parent.assertTrue(diff < 1E-3     , msg=f'''Max diff is {diff}'''       )


   def        snake_case					(			self     , _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase       ):
       A							:							Union[str, Any]				=    20
       A							:							List[Any]				=    model_class_name(__UpperCamelCase       )

       A							:							Optional[int]				=    jnp.concatenate(
           [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])       )]     , axis=-1     , )

       A							:							Optional[int]				=    model.init_cache(input_ids.shape[0]     , __UpperCamelCase       )
       A							:							List[Any]				=    jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1       )[None, :]     , (input_ids.shape[0], input_ids.shape[-1] - 1)       )

       A							:							List[str]				=    model(
           input_ids[:, :-1]     , attention_mask=__UpperCamelCase     , past_key_values=__UpperCamelCase     , position_ids=__UpperCamelCase     , )
       A							:							Tuple				=    jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]     , dtype='''i4'''       )
       A							:							Any				=    model(
           input_ids[:, -1:]     , past_key_values=outputs_cache.past_key_values     , attention_mask=__UpperCamelCase     , position_ids=__UpperCamelCase     , )

       A							:							Any				=    model(__UpperCamelCase     , attention_mask=__UpperCamelCase       )

       A							:							Optional[int]				=    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
       self.parent.assertTrue(diff < 1E-3     , msg=f'''Max diff is {diff}'''       )







@require_flax
class 	_lowercase    (				__UpperCamelCase		,       __UpperCamelCase		,       unittest.TestCase					):
   _UpperCamelCase									= (FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
   _UpperCamelCase									= (FlaxGPTaLMHeadModel,) if is_flax_available() else ()


   def        snake_case					(			self       ):
       A							:							List[Any]				=    FlaxGPTaModelTester(self       )


   def        snake_case					(			self       ):






       for model_class_name in self.all_model_classes:
           A							:							str				=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.check_use_cache_forward(__UpperCamelCase     , __UpperCamelCase     , __UpperCamelCase     , __UpperCamelCase       )


   def        snake_case					(			self       ):






       for model_class_name in self.all_model_classes:
           A							:							Tuple				=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.check_use_cache_forward_with_attn_mask(
               __UpperCamelCase     , __UpperCamelCase     , __UpperCamelCase     , __UpperCamelCase       )


   @slow
   def        snake_case					(			self       ):
       A							:							Dict				=    GPTaTokenizer.from_pretrained('''gpt2'''     , pad_token='''</s>'''     , padding_side='''left'''       )
       A							:							Dict				=    tokenizer(['''Hello this is a long string''', '''Hey''']     , return_tensors='''np'''     , padding=__UpperCamelCase     , truncation=__UpperCamelCase       )

       A							:							Union[str, Any]				=    FlaxGPTaLMHeadModel.from_pretrained('''gpt2'''       )
       A							:							Tuple				=    False
       A							:							Any				=    model.config.eos_token_id

       A							:							Union[str, Any]				=    jax.jit(model.generate       )

       A							:							Tuple				=    jit_generate(inputs['''input_ids''']     , attention_mask=inputs['''attention_mask''']       ).sequences

       A							:							Dict				=    tokenizer.batch_decode(__UpperCamelCase     , skip_special_tokens=__UpperCamelCase       )

       A							:							List[Any]				=    [
           """Hello this is a long string of words. I'm going to start with the first one.\n""",
           """Hey, I'm not sure if I'm going to be able to do""",
       ]

       self.assertListEqual(__UpperCamelCase     , __UpperCamelCase       )


   @is_pt_flax_cross_test
   def        snake_case					(			self       ):
       A							:							Optional[int]				=    self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__       ):
               # prepare inputs
               A							:							Optional[Any]				=    self._prepare_for_class(__UpperCamelCase     , __UpperCamelCase       )
               A							:							Tuple				=    {k: torch.tensor(v.tolist()       ) for k, v in prepared_inputs_dict.items()}

               # load corresponding PyTorch class
               A							:							List[Any]				=    model_class.__name__[4:]  # Skip the "Flax" at the beginning
               A							:							Optional[int]				=    getattr(__UpperCamelCase     , __UpperCamelCase       )

               A							:							Any				=    pt_inputs["""input_ids"""].shape
               A							:							List[Any]				=    np.random.randint(0     , seq_length - 1     , size=(batch_size,)       )
               for batch_idx, start_index in enumerate(__UpperCamelCase       ):
                   A							:							Union[str, Any]				=    0
                   A							:							Any				=    1
                   A							:							Dict				=    0
                   A							:							Dict				=    1
               A							:							int				=    pt_model_class(__UpperCamelCase       ).eval()
               A							:							Dict				=    model_class(__UpperCamelCase     , dtype=jnp.floataa       )

               A							:							Tuple				=    convert_pytorch_state_dict_to_flax(pt_model.state_dict()     , __UpperCamelCase       )
               A							:							List[str]				=    fx_state

               with torch.no_grad():
                   A							:							List[Any]				=    pt_model(**__UpperCamelCase       ).to_tuple()

               A							:							str				=    fx_model(**__UpperCamelCase       ).to_tuple()
               self.assertEqual(len(__UpperCamelCase       )     , len(__UpperCamelCase       )     , '''Output lengths differ between Flax and PyTorch'''       )
               for fx_output, pt_output in zip(__UpperCamelCase     , __UpperCamelCase       ):
                   self.assert_almost_equals(fx_output[:, -1]     , pt_output[:, -1].numpy()     , 4E-2       )

               with tempfile.TemporaryDirectory() as tmpdirname:
                   pt_model.save_pretrained(__UpperCamelCase       )
                   A							:							Union[str, Any]				=    model_class.from_pretrained(__UpperCamelCase     , from_pt=__UpperCamelCase       )

               A							:							int				=    fx_model_loaded(**__UpperCamelCase       ).to_tuple()
               self.assertEqual(
                   len(__UpperCamelCase       )     , len(__UpperCamelCase       )     , '''Output lengths differ between Flax and PyTorch'''       )
               for fx_output_loaded, pt_output in zip(__UpperCamelCase     , __UpperCamelCase       ):
                   self.assert_almost_equals(fx_output_loaded[:, -1]     , pt_output[:, -1].numpy()     , 4E-2       )


   @is_pt_flax_cross_test
   def        snake_case					(			self       ):
       A							:							Optional[int]				=    self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__       ):
               # prepare inputs
               A							:							Tuple				=    self._prepare_for_class(__UpperCamelCase     , __UpperCamelCase       )
               A							:							Optional[Any]				=    {k: torch.tensor(v.tolist()       ) for k, v in prepared_inputs_dict.items()}

               # load corresponding PyTorch class
               A							:							Optional[Any]				=    model_class.__name__[4:]  # Skip the "Flax" at the beginning
               A							:							Optional[Any]				=    getattr(__UpperCamelCase     , __UpperCamelCase       )

               A							:							List[Any]				=    pt_model_class(__UpperCamelCase       ).eval()
               A							:							Dict				=    model_class(__UpperCamelCase     , dtype=jnp.floataa       )

               A							:							int				=    load_flax_weights_in_pytorch_model(__UpperCamelCase     , fx_model.params       )
               A							:							Tuple				=    pt_inputs["""input_ids"""].shape
               A							:							Optional[Any]				=    np.random.randint(0     , seq_length - 1     , size=(batch_size,)       )
               for batch_idx, start_index in enumerate(__UpperCamelCase       ):
                   A							:							List[str]				=    0
                   A							:							Union[str, Any]				=    1
                   A							:							Tuple				=    0
                   A							:							Optional[int]				=    1

               # make sure weights are tied in PyTorch
               pt_model.tie_weights()

               with torch.no_grad():
                   A							:							Tuple				=    pt_model(**__UpperCamelCase       ).to_tuple()

               A							:							Optional[Any]				=    fx_model(**__UpperCamelCase       ).to_tuple()
               self.assertEqual(len(__UpperCamelCase       )     , len(__UpperCamelCase       )     , '''Output lengths differ between Flax and PyTorch'''       )
               for fx_output, pt_output in zip(__UpperCamelCase     , __UpperCamelCase       ):
                   self.assert_almost_equals(fx_output[:, -1]     , pt_output[:, -1].numpy()     , 4E-2       )

               with tempfile.TemporaryDirectory() as tmpdirname:
                   fx_model.save_pretrained(__UpperCamelCase       )
                   A							:							str				=    pt_model_class.from_pretrained(__UpperCamelCase     , from_flax=__UpperCamelCase       )

               with torch.no_grad():
                   A							:							Any				=    pt_model_loaded(**__UpperCamelCase       ).to_tuple()

               self.assertEqual(
                   len(__UpperCamelCase       )     , len(__UpperCamelCase       )     , '''Output lengths differ between Flax and PyTorch'''       )
               for fx_output, pt_output in zip(__UpperCamelCase     , __UpperCamelCase       ):
                   self.assert_almost_equals(fx_output[:, -1]     , pt_output[:, -1].numpy()     , 4E-2       )


   @slow
   def        snake_case					(			self       ):






       for model_class_name in self.all_model_classes:
           A							:							str				=    model_class_name.from_pretrained('''gpt2'''     , from_pt=__UpperCamelCase       )
           A							:							List[Any]				=    model(np.ones((1, 1)       )       )
           self.assertIsNotNone(__UpperCamelCase       )


'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class  SCREAMING_SNAKE_CASE__       (  __UpperCamelCase			,					unittest.TestCase   ):
	lowercase__	    =							GPTNeoXJapaneseTokenizer
	lowercase__	    =							False
	lowercase__	    =							{"do_clean_text": False, "add_prefix_space": False}
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					super().setUp()

					__a       :     Optional[int]								=	[
					    """こん""",
					    """こんに""",
					    """にちは""",
					    """ばんは""",
					    """世界,㔺界""",
					    """、""",
					    """。""",
					    """<BR>""",
					    """<SP>""",
					    """<TAB>""",
					    """<URL>""",
					    """<EMAIL>""",
					    """<TEL>""",
					    """<DATE>""",
					    """<PRICE>""",
					    """<BLOCK>""",
					    """<KIGOU>""",
					    """<U2000U2BFF>""",
					    """<|emoji1|>""",
					    """<unk>""",
					    """<|startoftext|>""",
					    """<|endoftext|>""",
					]
					__a       :     Any								=	{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
					__a       :     List[str]								=	{"""unk_token""": """<unk>"""}

					__a       :     Any								=	os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES["""vocab_file"""]			)
					__a       :     Tuple								=	os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES["""emoji_file"""]			)
					with open(self.vocab_file  ,    """w"""  ,    encoding="""utf-8"""			) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)
					with open(self.emoji_file  ,    """w"""			) as emoji_writer:
									emoji_writer.write(json.dumps(__UpperCamelCase			)			)
	def __lowerCamelCase    (				self  ,    **__UpperCamelCase			):



					'''simple docstring'''






					kwargs.update(self.special_tokens_map			)
					return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname  ,    **__UpperCamelCase			)
	def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					__a       :     Any								=	"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
					__a       :     Optional[int]								=	"""こんにちは、世界。 \nこんばんは、世界。😀"""
					return input_text, output_text
	def __lowerCamelCase    (				self  ,    __UpperCamelCase			):



					'''simple docstring'''






					__a							,	__a       :     List[str]								=	self.get_input_output_texts(__UpperCamelCase			)
					__a       :     List[str]								=	tokenizer.encode(__UpperCamelCase  ,    add_special_tokens=__UpperCamelCase			)
					__a       :     str								=	tokenizer.decode(__UpperCamelCase  ,    clean_up_tokenization_spaces=__UpperCamelCase			)
					return text, ids
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					pass  # TODO add if relevant
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					pass  # TODO add if relevant
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					pass  # TODO add if relevant
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[Any]								=	self.get_tokenizer()

					# Testing tokenization
					__a       :     Dict								=	"""こんにちは、世界。　こんばんは、㔺界。"""
					__a       :     Optional[int]								=	["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
					__a       :     str								=	tokenizer.tokenize(__UpperCamelCase			)
					self.assertListEqual(__UpperCamelCase  ,    __UpperCamelCase			)

					# Testing conversion to ids without special tokens
					__a       :     Optional[Any]								=	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
					__a       :     Optional[int]								=	tokenizer.convert_tokens_to_ids(__UpperCamelCase			)
					self.assertListEqual(__UpperCamelCase  ,    __UpperCamelCase			)

					# Testing conversion to ids with special tokens
					__a       :     List[str]								=	tokens + [tokenizer.unk_token]
					__a       :     Dict								=	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
					__a       :     Any								=	tokenizer.convert_tokens_to_ids(__UpperCamelCase			)
					self.assertListEqual(__UpperCamelCase  ,    __UpperCamelCase			)
	@slow
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     int								=	self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""			)

					__a       :     Union[str, Any]								=	tokenizer.encode("""ありがとう。"""  ,    add_special_tokens=__UpperCamelCase			)
					__a       :     Dict								=	tokenizer.encode("""どういたしまして。"""  ,    add_special_tokens=__UpperCamelCase			)

					__a       :     Optional[Any]								=	tokenizer.build_inputs_with_special_tokens(__UpperCamelCase			)
					__a       :     int								=	tokenizer.build_inputs_with_special_tokens(__UpperCamelCase  ,    __UpperCamelCase			)

					assert encoded_sentence == ids_a
					assert encoded_pair == ids_a + ids_a
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					pass

	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					pass