
def      _a     (   __UpperCamelCase       :  int    ):
			if not isinstance(__UpperCamelCase	,__UpperCamelCase    ):
						raise ValueError('''double_factorial() only accepts integral values'''    )
			if n < 0:
						raise ValueError('''double_factorial() not defined for negative values'''    )
			return 1 if n <= 1 else n * double_factorial(n - 2    )


if __name__ == "__main__":
							import doctest

							doctest.testmod()





import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import OPTConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
							import os

							# The slow tests are often failing with OOM error on GPU
							# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
							# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
							A__       :   Tuple   						=					"""platform"""

							import jax
							import jax.numpy as jnp

							from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer



def      _a     (   __UpperCamelCase       :  Optional[Any]	,__UpperCamelCase       :  Optional[Any]	,__UpperCamelCase       :  Union[str, Any]=None	,__UpperCamelCase       :  Optional[int]=None    ):
			if attention_mask is None:
						lowerCAmelCase__ :    Any				      =			np.where(input_ids != config.pad_token_id	,1	,0    )
			return {
			    "input_ids": input_ids,
			    "attention_mask": attention_mask,
			}





@require_flax
class 						lowercase       :
					def __init__(		self ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__=13 ,	SCREAMING_SNAKE_CASE__=7 ,	SCREAMING_SNAKE_CASE__=True ,	SCREAMING_SNAKE_CASE__=False ,	SCREAMING_SNAKE_CASE__=99 ,	SCREAMING_SNAKE_CASE__=16 ,	SCREAMING_SNAKE_CASE__=2 ,	SCREAMING_SNAKE_CASE__=4 ,	SCREAMING_SNAKE_CASE__=4 ,	SCREAMING_SNAKE_CASE__="gelu" ,	SCREAMING_SNAKE_CASE__=0.1 ,	SCREAMING_SNAKE_CASE__=0.1 ,	SCREAMING_SNAKE_CASE__=20 ,	SCREAMING_SNAKE_CASE__=2 ,	SCREAMING_SNAKE_CASE__=1 ,	SCREAMING_SNAKE_CASE__=0 ,	SCREAMING_SNAKE_CASE__=16 ,	SCREAMING_SNAKE_CASE__=16 ,	SCREAMING_SNAKE_CASE__=0.02 ,	):
								"""simple docstring"""







								lowerCAmelCase__ :    List[str]				      =			parent
								lowerCAmelCase__ :    Optional[Any]				      =			batch_size
								lowerCAmelCase__ :    Any				      =			seq_length
								lowerCAmelCase__ :    str				      =			is_training
								lowerCAmelCase__ :    List[Any]				      =			use_labels
								lowerCAmelCase__ :    List[Any]				      =			vocab_size
								lowerCAmelCase__ :    List[str]				      =			hidden_size
								lowerCAmelCase__ :    str				      =			num_hidden_layers
								lowerCAmelCase__ :    List[Any]				      =			num_attention_heads
								lowerCAmelCase__ :    Optional[Any]				      =			intermediate_size
								lowerCAmelCase__ :    int				      =			hidden_act
								lowerCAmelCase__ :    Optional[int]				      =			hidden_dropout_prob
								lowerCAmelCase__ :    Any				      =			attention_probs_dropout_prob
								lowerCAmelCase__ :    str				      =			max_position_embeddings
								lowerCAmelCase__ :    List[str]				      =			eos_token_id
								lowerCAmelCase__ :    List[Any]				      =			pad_token_id
								lowerCAmelCase__ :    Tuple				      =			bos_token_id
								lowerCAmelCase__ :    str				      =			embed_dim
								lowerCAmelCase__ :    Optional[int]				      =			word_embed_proj_dim
								lowerCAmelCase__ :    List[Any]				      =			initializer_range
								lowerCAmelCase__ :    Tuple				      =			False
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Optional[int]				      =			np.clip(ids_tensor([self.batch_size, self.seq_length - 1] ,	self.vocab_size    ) ,	3 ,	self.vocab_size    )
								lowerCAmelCase__ :    Union[str, Any]				      =			np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1) ,	dtype=np.intaa    )) ,	-1    )

								lowerCAmelCase__ :    Union[str, Any]				      =			OPTConfig(
								    vocab_size=self.vocab_size ,	hidden_size=self.hidden_size ,	num_hidden_layers=self.num_hidden_layers ,	num_attention_heads=self.num_attention_heads ,	ffn_dim=self.intermediate_size ,	dropout=self.hidden_dropout_prob ,	attention_dropout=self.attention_probs_dropout_prob ,	max_position_embeddings=self.max_position_embeddings ,	eos_token_id=self.eos_token_id ,	bos_token_id=self.bos_token_id ,	pad_token_id=self.pad_token_id ,	embed_dim=self.embed_dim ,	is_encoder_decoder=SCREAMING_SNAKE_CASE__ ,	word_embed_proj_dim=self.word_embed_proj_dim ,	initializer_range=self.initializer_range ,	use_cache=SCREAMING_SNAKE_CASE__ ,	)
								lowerCAmelCase__ :    Optional[int]				      =			prepare_opt_inputs_dict(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
								return config, inputs_dict
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__      ,				lowerCAmelCase__ :    List[str]				      =			self.prepare_config_and_inputs()
								return config, inputs_dict
					def lowercase_			(		self ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Any				      =			20
								lowerCAmelCase__ :    Optional[int]				      =			model_class_name(SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    Optional[Any]				      =			inputs_dict['''input_ids''']
								lowerCAmelCase__ :    Optional[Any]				      =			inputs_dict['''attention_mask''']

								lowerCAmelCase__ :    Tuple				      =			model.init_cache(input_ids.shape[0] ,	SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Tuple				      =			jnp.ones((input_ids.shape[0], max_length) ,	dtype='''i4'''    )

								lowerCAmelCase__ :    Any				      =			jnp.broadcast_to(
								    jnp.arange(input_ids.shape[-1] - 1    )[None, :] ,	(input_ids.shape[0], input_ids.shape[-1] - 1) ,	)
								lowerCAmelCase__ :    str				      =			model(
								    input_ids[:, :-1] ,	attention_mask=SCREAMING_SNAKE_CASE__ ,	past_key_values=SCREAMING_SNAKE_CASE__ ,	position_ids=SCREAMING_SNAKE_CASE__ ,	)

								lowerCAmelCase__ :    int				      =			jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]] ,	dtype='''i4'''    )
								lowerCAmelCase__ :    List[str]				      =			model(
								    input_ids[:, -1:] ,	attention_mask=SCREAMING_SNAKE_CASE__ ,	past_key_values=outputs_cache.past_key_values ,	position_ids=SCREAMING_SNAKE_CASE__ ,	)

								lowerCAmelCase__ :    List[str]				      =			model(SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    List[Any]				      =			np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])    )    )
								self.parent.assertTrue(diff < 1E-3 ,	msg=f'''Max diff is {diff}'''    )
					def lowercase_			(		self ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    ):
								"""simple docstring"""







								lowerCAmelCase__ :    int				      =			20
								lowerCAmelCase__ :    Optional[Any]				      =			model_class_name(SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__      ,				lowerCAmelCase__ :    Optional[Any]				      =			(
								    inputs_dict['''input_ids'''],
								    inputs_dict['''attention_mask'''],
								)

								lowerCAmelCase__ :    Optional[Any]				      =			jnp.concatenate(
								    [
								        attention_mask,
								        jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])    ),
								    ] ,	axis=-1 ,	)

								lowerCAmelCase__ :    Any				      =			model.init_cache(input_ids.shape[0] ,	SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Union[str, Any]				      =			jnp.broadcast_to(
								    jnp.arange(input_ids.shape[-1] - 1    )[None, :] ,	(input_ids.shape[0], input_ids.shape[-1] - 1) ,	)

								lowerCAmelCase__ :    Tuple				      =			model(
								    input_ids[:, :-1] ,	attention_mask=SCREAMING_SNAKE_CASE__ ,	past_key_values=SCREAMING_SNAKE_CASE__ ,	position_ids=SCREAMING_SNAKE_CASE__ ,	)
								lowerCAmelCase__ :    int				      =			jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]] ,	dtype='''i4'''    )
								lowerCAmelCase__ :    List[Any]				      =			model(
								    input_ids[:, -1:] ,	past_key_values=outputs_cache.past_key_values ,	attention_mask=SCREAMING_SNAKE_CASE__ ,	position_ids=SCREAMING_SNAKE_CASE__ ,	)

								lowerCAmelCase__ :    Any				      =			model(SCREAMING_SNAKE_CASE__ ,	attention_mask=SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    int				      =			np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])    )    )
								self.parent.assertTrue(diff < 1E-3 ,	msg=f'''Max diff is {diff}'''    )





@require_flax
class 						lowercase       (  __UpperCamelCase	,		unittest.TestCase	,		__UpperCamelCase					):
					__a					=    (FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
					__a					=    () if is_flax_available() else ()
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Union[str, Any]				      =			FlaxOPTModelTester(self    )
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__      ,				lowerCAmelCase__ :    List[Any]				      =			self.model_tester.prepare_config_and_inputs()
								for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__      ,				lowerCAmelCase__ :    str				      =			self.model_tester.prepare_config_and_inputs()
								for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward_with_attn_mask(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
					@slow
					def lowercase_			(		self    ):
								"""simple docstring"""







								for model_class_name in self.all_model_classes:
											lowerCAmelCase__ :    Tuple				      =			model_class_name.from_pretrained('''facebook/opt-125m'''    )
											lowerCAmelCase__ :    Dict				      =			np.ones((1, 1)    ) * model.config.eos_token_id
											lowerCAmelCase__ :    List[str]				      =			model(SCREAMING_SNAKE_CASE__    )
											self.assertIsNotNone(SCREAMING_SNAKE_CASE__    )





@require_sentencepiece
@require_flax
class 						lowercase       (  unittest.TestCase					):
					@slow
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Optional[Any]				      =			FlaxOPTModel.from_pretrained('''facebook/opt-350m'''    )
								lowerCAmelCase__ :    List[Any]				      =			jnp.array([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]    )
								lowerCAmelCase__ :    Union[str, Any]				      =			model(input_ids=SCREAMING_SNAKE_CASE__    ).last_hidden_state
								lowerCAmelCase__ :    Any				      =			(1, 11, 512)
								self.assertEqual(output.shape ,	SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    int				      =			jnp.array(
								    [[-0.2_867, -1.9_256, -0.3_062], [-1.2_711, -0.1_337, -0.1_897], [0.4_109, 0.1_187, -1.3_142]]    )
								self.assertTrue(jnp.allclose(output[:, :3, :3] ,	SCREAMING_SNAKE_CASE__ ,	atol=4E-2    )    )





@require_flax
@slow
class 						lowercase       (  unittest.TestCase					):
					def lowercase_			(		self    ):
								"""simple docstring"""







								super().setUp()
								lowerCAmelCase__ :    Union[str, Any]				      =			'''facebook/opt-350m'''
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Tuple				      =			FlaxOPTForCausalLM.from_pretrained(self.path_model    )
								lowerCAmelCase__ :    Any				      =			GPTaTokenizer.from_pretrained(self.path_model    )

								lowerCAmelCase__ :    Tuple				      =			[
								    '''Today is a beautiful day and I want to''',
								    '''In the city of''',
								    '''Paris is the capital of France and''',
								    '''Computers and mobile phones have taken''',
								]
								# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
								lowerCAmelCase__ :    Dict				      =			tokenizer(SCREAMING_SNAKE_CASE__ ,	return_tensors='''jax''' ,	padding=SCREAMING_SNAKE_CASE__ ,	add_special_tokens=SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Optional[Any]				      =			model(inputs.input_ids ,	attention_mask=inputs.attention_mask    )[0].mean(axis=-1    )
								lowerCAmelCase__ :    Optional[Any]				      =			jnp.array(
								    [
								        [1.3_851, -13.8_923, -10.5_229, -10.7_533, -0.2_309, -10.2_384, -0.5_365, -9.0_947, -5.1_670],
								        [-4.7_073, -10.6_276, -3.9_415, -21.5_242, -0.2_822, -0.2_822, -0.2_822, -0.2_822, -0.2_822],
								        [0.6_247, -3.4_229, -8.9_179, -1.4_297, -14.1_650, 1.4_146, -9.0_218, -0.2_703, -0.2_703],
								        [6.4_783, -1.9_913, -10.7_926, -2.3_336, 1.5_092, -0.9_974, -6.8_213, 1.3_477, 1.3_477],
								    ]    )
								self.assertTrue(jnp.allclose(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	atol=4E-2    )    )

								lowerCAmelCase__ :    List[Any]				      =			jax.jit(SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    List[Any]				      =			model(inputs.input_ids ,	attention_mask=inputs.attention_mask    )[0].mean(axis=-1    )
								self.assertTrue(jnp.allclose(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__ ,	atol=4E-2    )    )





@require_flax
@slow
class 						lowercase       (  unittest.TestCase					):
					@property
					def lowercase_			(		self    ):
								"""simple docstring"""







								return [
								    "Today is a beautiful day and I want",
								    "In the city of",
								    "Paris is the capital of France and",
								    "Computers and mobile phones have taken",
								]
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Dict				      =			'''facebook/opt-125m'''

								lowerCAmelCase__ :    List[Any]				      =			[
								    '''Today is a beautiful day and I want to''',
								    '''In the city of New York, the city''',
								    '''Paris is the capital of France and the capital''',
								    '''Computers and mobile phones have taken over the''',
								]

								lowerCAmelCase__ :    Any				      =			[]

								lowerCAmelCase__ :    Optional[int]				      =			FlaxOPTForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    List[Any]				      =			GPTaTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__    )

								for prompt in self.prompts:
											lowerCAmelCase__ :    Union[str, Any]				      =			tokenizer(SCREAMING_SNAKE_CASE__ ,	return_tensors='''jax'''    ).input_ids

											lowerCAmelCase__ :    Dict				      =			model.generate(SCREAMING_SNAKE_CASE__ ,	max_length=10    )
											lowerCAmelCase__ :    Tuple				      =			generated_ids[0]

											lowerCAmelCase__ :    List[str]				      =			tokenizer.batch_decode(SCREAMING_SNAKE_CASE__ ,	skip_special_tokens=SCREAMING_SNAKE_CASE__    )
											predicted_outputs += generated_string

								self.assertListEqual(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    List[Any]				      =			'''facebook/opt-350m'''

								lowerCAmelCase__ :    Dict				      =			[
								    '''Today is a beautiful day and I want to''',
								    '''In the city of San Francisco, the city''',
								    '''Paris is the capital of France and the capital''',
								    '''Computers and mobile phones have taken over the''',
								]

								lowerCAmelCase__ :    Any				      =			[]
								lowerCAmelCase__ :    List[str]				      =			FlaxOPTForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    int				      =			GPTaTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__    )

								for prompt in self.prompts:
											lowerCAmelCase__ :    str				      =			tokenizer(SCREAMING_SNAKE_CASE__ ,	return_tensors='''jax'''    ).input_ids

											lowerCAmelCase__ :    Optional[Any]				      =			model.generate(SCREAMING_SNAKE_CASE__ ,	max_length=10    )
											lowerCAmelCase__ :    List[Any]				      =			generated_ids[0]

											lowerCAmelCase__ :    Optional[Any]				      =			tokenizer.batch_decode(SCREAMING_SNAKE_CASE__ ,	skip_special_tokens=SCREAMING_SNAKE_CASE__    )
											predicted_outputs += generated_string

								self.assertListEqual(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Optional[Any]				      =			'''facebook/opt-125m'''
								lowerCAmelCase__ :    Dict				      =			[
								    '''Today is a beautiful day and I want to thank''',
								    '''In the city of Rome Canaver Canaver Canaver Canaver''',
								]
								lowerCAmelCase__ :    Tuple				      =			FlaxOPTForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Optional[Any]				      =			GPTaTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    int				      =			tokenizer(
								    [
								        '''Today is a beautiful day and I want to''',
								        '''In the city of''',
								    ] ,	return_tensors='''jax''' ,	padding=SCREAMING_SNAKE_CASE__ ,	)

								lowerCAmelCase__ :    Union[str, Any]				      =			jax.jit(model.generate    )

								lowerCAmelCase__ :    Optional[Any]				      =			jit_generate(inputs['''input_ids'''] ,	attention_mask=inputs['''attention_mask''']    ).sequences

								lowerCAmelCase__ :    str				      =			tokenizer.batch_decode(SCREAMING_SNAKE_CASE__ ,	skip_special_tokens=SCREAMING_SNAKE_CASE__    )

								self.assertIsNotNone(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
					def lowercase_			(		self    ):
								"""simple docstring"""







								lowerCAmelCase__ :    Any				      =			'''facebook/opt-350m'''

								lowerCAmelCase__ :    Tuple				      =			GPTaTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    int				      =			FlaxOPTForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    Union[str, Any]				      =			'''left'''

								# use different length sentences to test batching
								lowerCAmelCase__ :    Optional[int]				      =			[
								    '''Hello, my dog is a little''',
								    '''Today, I''',
								]

								lowerCAmelCase__ :    Optional[int]				      =			tokenizer(SCREAMING_SNAKE_CASE__ ,	return_tensors='''jax''' ,	padding=SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Any				      =			inputs['''input_ids''']

								lowerCAmelCase__ :    Optional[Any]				      =			model.generate(input_ids=SCREAMING_SNAKE_CASE__ ,	attention_mask=inputs['''attention_mask'''] ,	trace=SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    List[Any]				      =			tokenizer(sentences[0] ,	return_tensors='''jax'''    ).input_ids
								lowerCAmelCase__ :    List[Any]				      =			model.generate(input_ids=SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    Dict				      =			inputs_non_padded.shape[-1] - inputs['''attention_mask'''][-1].sum()
								lowerCAmelCase__ :    Union[str, Any]				      =			tokenizer(sentences[1] ,	return_tensors='''jax'''    ).input_ids
								lowerCAmelCase__ :    Dict				      =			model.generate(input_ids=SCREAMING_SNAKE_CASE__ ,	max_length=model.config.max_length - num_paddings    )

								lowerCAmelCase__ :    Dict				      =			tokenizer.batch_decode(outputs[0] ,	skip_special_tokens=SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Dict				      =			tokenizer.decode(output_non_padded[0][0] ,	skip_special_tokens=SCREAMING_SNAKE_CASE__    )
								lowerCAmelCase__ :    Tuple				      =			tokenizer.decode(output_padded[0][0] ,	skip_special_tokens=SCREAMING_SNAKE_CASE__    )

								lowerCAmelCase__ :    int				      =			[
								    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
								    '''Today, I was in the middle of a conversation with a friend about the''',
								]
								self.assertListEqual(SCREAMING_SNAKE_CASE__ ,	SCREAMING_SNAKE_CASE__    )
								self.assertListEqual(SCREAMING_SNAKE_CASE__ ,	[non_padded_sentence, padded_sentence]    )



