





"""simple docstring"""





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel





class 				_lowerCamelCase					:


						def __init__(self      ,							__a      ,							__a=13      ,							__a=7      ,							__a=True      ,							__a=True      ,							__a=False      ,							__a=True      ,							__a=99      ,							__a=32      ,							__a=5      ,							__a=4      ,							__a=37      ,							__a="gelu"      ,							__a=0.1      ,							__a=0.1      ,							__a=5_12      ,							__a=16      ,							__a=2      ,							__a=0.02      ,							__a=3      ,							__a=4      ,							__a=None      ,							)       ->      List[Any]:
								UpperCamelCase      =		parent
								UpperCamelCase      =		batch_size
								UpperCamelCase      =		seq_length
								UpperCamelCase      =		is_training
								UpperCamelCase      =		use_input_mask
								UpperCamelCase      =		use_token_type_ids
								UpperCamelCase      =		use_labels
								UpperCamelCase      =		vocab_size
								UpperCamelCase      =		hidden_size
								UpperCamelCase      =		num_hidden_layers
								UpperCamelCase      =		num_attention_heads
								UpperCamelCase      =		intermediate_size
								UpperCamelCase      =		hidden_act
								UpperCamelCase      =		hidden_dropout_prob
								UpperCamelCase      =		attention_probs_dropout_prob
								UpperCamelCase      =		max_position_embeddings
								UpperCamelCase      =		type_vocab_size
								UpperCamelCase      =		type_sequence_label_size
								UpperCamelCase      =		initializer_range
								UpperCamelCase      =		num_labels
								UpperCamelCase      =		num_choices
								UpperCamelCase      =		scope


						def 			snake_case_      (self						)       ->      Optional[Any]:
								UpperCamelCase      =		ids_tensor([self.batch_size, self.seq_length]      ,							self.vocab_size						)

								UpperCamelCase      =		None
								if self.use_input_mask:
										UpperCamelCase      =		random_attention_mask([self.batch_size, self.seq_length]						)

								UpperCamelCase      =		None
								if self.use_token_type_ids:
										UpperCamelCase      =		ids_tensor([self.batch_size, self.seq_length]      ,							self.type_vocab_size						)

								UpperCamelCase      =		None
								UpperCamelCase      =		None
								UpperCamelCase      =		None
								if self.use_labels:
										UpperCamelCase      =		ids_tensor([self.batch_size]      ,							self.type_sequence_label_size						)
										UpperCamelCase      =		ids_tensor([self.batch_size, self.seq_length]      ,							self.num_labels						)
										UpperCamelCase      =		ids_tensor([self.batch_size]      ,							self.num_choices						)

								UpperCamelCase      =		self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def 			snake_case_      (self						)       ->      List[str]:
								return OpenLlamaConfig(
								    vocab_size=self.vocab_size      ,							hidden_size=self.hidden_size      ,							num_hidden_layers=self.num_hidden_layers      ,							num_attention_heads=self.num_attention_heads      ,							intermediate_size=self.intermediate_size      ,							hidden_act=self.hidden_act      ,							hidden_dropout_prob=self.hidden_dropout_prob      ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,							max_position_embeddings=self.max_position_embeddings      ,							type_vocab_size=self.type_vocab_size      ,							is_decoder=lowercase_      ,							initializer_range=self.initializer_range      ,							use_stable_embedding=lowercase_      ,							)


						def 			snake_case_      (self      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a						)       ->      List[str]:
								UpperCamelCase      =		OpenLlamaModel(config=lowercase_						)
								model.to(lowercase_						)
								model.eval()
								UpperCamelCase      =		model(lowercase_      ,							attention_mask=lowercase_						)
								UpperCamelCase      =		model(lowercase_						)
								self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, self.seq_length, self.hidden_size)						)


						def 			snake_case_      (self      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							)       ->      Any:
								UpperCamelCase      =		True
								UpperCamelCase      =		OpenLlamaModel(lowercase_						)
								model.to(lowercase_						)
								model.eval()
								UpperCamelCase      =		model(
								    lowercase_      ,							attention_mask=lowercase_      ,							encoder_hidden_states=lowercase_      ,							encoder_attention_mask=lowercase_      ,							)
								UpperCamelCase      =		model(
								    lowercase_      ,							attention_mask=lowercase_      ,							encoder_hidden_states=lowercase_      ,							)
								UpperCamelCase      =		model(lowercase_      ,							attention_mask=lowercase_						)
								self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, self.seq_length, self.hidden_size)						)


						def 			snake_case_      (self      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							)       ->      List[str]:
								UpperCamelCase      =		OpenLlamaForCausalLM(config=lowercase_						)
								model.to(lowercase_						)
								model.eval()
								UpperCamelCase      =		model(lowercase_      ,							attention_mask=lowercase_      ,							labels=lowercase_						)
								self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.seq_length, self.vocab_size)						)


						def 			snake_case_      (self      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							__a      ,							)       ->      List[Any]:
								UpperCamelCase      =		True
								UpperCamelCase      =		True
								UpperCamelCase      =		OpenLlamaForCausalLM(config=lowercase_						)
								model.to(lowercase_						)
								model.eval()

								# first forward pass
								UpperCamelCase      =		model(
								    lowercase_      ,							attention_mask=lowercase_      ,							encoder_hidden_states=lowercase_      ,							encoder_attention_mask=lowercase_      ,							use_cache=lowercase_      ,							)
								UpperCamelCase      =		outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								UpperCamelCase      =		ids_tensor((self.batch_size, 3)      ,							config.vocab_size						)
								UpperCamelCase      =		ids_tensor((self.batch_size, 3)      ,							vocab_size=2						)

								# append to next input_ids and
								UpperCamelCase      =		torch.cat([input_ids, next_tokens]      ,							dim=-1						)
								UpperCamelCase      =		torch.cat([input_mask, next_mask]      ,							dim=-1						)

								UpperCamelCase      =		model(
								    lowercase_      ,							attention_mask=lowercase_      ,							encoder_hidden_states=lowercase_      ,							encoder_attention_mask=lowercase_      ,							output_hidden_states=lowercase_      ,							)["hidden_states"][0]
								UpperCamelCase      =		model(
								    lowercase_      ,							attention_mask=lowercase_      ,							encoder_hidden_states=lowercase_      ,							encoder_attention_mask=lowercase_      ,							past_key_values=lowercase_      ,							output_hidden_states=lowercase_      ,							)["hidden_states"][0]

								# select random slice
								UpperCamelCase      =		ids_tensor((1,)      ,							output_from_past.shape[-1]						).item()
								UpperCamelCase      =		output_from_no_past[:, -3:, random_slice_idx].detach()
								UpperCamelCase      =		output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(lowercase_      ,							lowercase_      ,							atol=1e-3						)						)




						def 			snake_case_      (self						)       ->      Optional[int]:
								UpperCamelCase      =		self.prepare_config_and_inputs()
								(
								    (
								    UpperCamelCase      
								)      , (
								    UpperCamelCase      
								)      , (
								    UpperCamelCase      
								)      , (
								    UpperCamelCase      
								)      , (
								    UpperCamelCase      
								)      , (
								    UpperCamelCase      
								)      , (
								    UpperCamelCase      
								)      ,       
								)      =		config_and_inputs
								UpperCamelCase      =		{"input_ids": input_ids, "attention_mask": input_mask}
								return config, inputs_dict





@require_torch
class 				_lowerCamelCase					(				lowerCamelCase__     ,	lowerCamelCase__     ,	lowerCamelCase__     ,	unittest.TestCase       ):
						UpperCAmelCase_       =	(
						    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
						)
						UpperCAmelCase_       =	(OpenLlamaForCausalLM,) if is_torch_available() else ()
						UpperCAmelCase_       =	(
						    {
						        'feature-extraction': OpenLlamaModel,
						        'text-classification': OpenLlamaForSequenceClassification,
						        'text-generation': OpenLlamaForCausalLM,
						        'zero-shot': OpenLlamaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						UpperCAmelCase_       =	False
						UpperCAmelCase_       =	False


						def 			snake_case_      (self						)       ->      List[str]:
								UpperCamelCase      =		OpenLlamaModelTester(self						)
								UpperCamelCase      =		ConfigTester(self      ,							config_class=lowercase_      ,							hidden_size=37						)


						def 			snake_case_      (self						)       ->      str:
								self.config_tester.run_common_tests()


						def 			snake_case_      (self						)       ->      Optional[Any]:
								UpperCamelCase      =		self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowercase_						)


						def 			snake_case_      (self						)       ->      str:
								UpperCamelCase      =		self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
										UpperCamelCase      =		type
										self.model_tester.create_and_check_model(*lowercase_						)


						def 			snake_case_      (self						)       ->      int:
								UpperCamelCase      , UpperCamelCase      =		self.model_tester.prepare_config_and_inputs_for_common()
								UpperCamelCase      =		3
								UpperCamelCase      =		input_dict["input_ids"]
								UpperCamelCase      =		input_ids.ne(1						).to(lowercase_						)
								UpperCamelCase      =		ids_tensor([self.model_tester.batch_size]      ,							self.model_tester.type_sequence_label_size						)
								UpperCamelCase      =		OpenLlamaForSequenceClassification(lowercase_						)
								model.to(lowercase_						)
								model.eval()
								UpperCamelCase      =		model(lowercase_      ,							attention_mask=lowercase_      ,							labels=lowercase_						)
								self.assertEqual(result.logits.shape      ,							(self.model_tester.batch_size, self.model_tester.num_labels)						)


						def 			snake_case_      (self						)       ->      Any:
								UpperCamelCase      , UpperCamelCase      =		self.model_tester.prepare_config_and_inputs_for_common()
								UpperCamelCase      =		3
								UpperCamelCase      =		"single_label_classification"
								UpperCamelCase      =		input_dict["input_ids"]
								UpperCamelCase      =		input_ids.ne(1						).to(lowercase_						)
								UpperCamelCase      =		ids_tensor([self.model_tester.batch_size]      ,							self.model_tester.type_sequence_label_size						)
								UpperCamelCase      =		OpenLlamaForSequenceClassification(lowercase_						)
								model.to(lowercase_						)
								model.eval()
								UpperCamelCase      =		model(lowercase_      ,							attention_mask=lowercase_      ,							labels=lowercase_						)
								self.assertEqual(result.logits.shape      ,							(self.model_tester.batch_size, self.model_tester.num_labels)						)


						def 			snake_case_      (self						)       ->      Optional[int]:
								UpperCamelCase      , UpperCamelCase      =		self.model_tester.prepare_config_and_inputs_for_common()
								UpperCamelCase      =		3
								UpperCamelCase      =		"multi_label_classification"
								UpperCamelCase      =		input_dict["input_ids"]
								UpperCamelCase      =		input_ids.ne(1						).to(lowercase_						)
								UpperCamelCase      =		ids_tensor(
								    [self.model_tester.batch_size, config.num_labels]      ,							self.model_tester.type_sequence_label_size						).to(torch.float						)
								UpperCamelCase      =		OpenLlamaForSequenceClassification(lowercase_						)
								model.to(lowercase_						)
								model.eval()
								UpperCamelCase      =		model(lowercase_      ,							attention_mask=lowercase_      ,							labels=lowercase_						)
								self.assertEqual(result.logits.shape      ,							(self.model_tester.batch_size, self.model_tester.num_labels)						)


						@unittest.skip("Open-Llama buffers include complex numbers, which breaks this test"						)
						def 			snake_case_      (self						)       ->      List[str]:
								pass




						@parameterized.expand([("linear",), ("dynamic",)]						)
						def 			snake_case_      (self      ,							__a						)       ->      Optional[Any]:
								UpperCamelCase      , UpperCamelCase      =		self.model_tester.prepare_config_and_inputs_for_common()
								UpperCamelCase      =		ids_tensor([1, 10]      ,							config.vocab_size						)
								UpperCamelCase      =		ids_tensor([1, int(config.max_position_embeddings * 1.5						)]      ,							config.vocab_size						)

								set_seed(42						)  # Fixed seed at init time so the two models get the same random weights
								UpperCamelCase      =		OpenLlamaModel(lowercase_						)
								original_model.to(lowercase_						)
								original_model.eval()
								UpperCamelCase      =		original_model(lowercase_						).last_hidden_state
								UpperCamelCase      =		original_model(lowercase_						).last_hidden_state

								set_seed(42						)  # Fixed seed at init time so the two models get the same random weights
								UpperCamelCase      =		{"type": scaling_type, "factor": 10.0}
								UpperCamelCase      =		OpenLlamaModel(lowercase_						)
								scaled_model.to(lowercase_						)
								scaled_model.eval()
								UpperCamelCase      =		scaled_model(lowercase_						).last_hidden_state
								UpperCamelCase      =		scaled_model(lowercase_						).last_hidden_state

								# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
								# maximum sequence length, so the outputs for the short input should match.
								if scaling_type == "dynamic":
										self.assertTrue(torch.allclose(lowercase_      ,							lowercase_      ,							atol=1e-5						)						)
								else:
										self.assertFalse(torch.allclose(lowercase_      ,							lowercase_      ,							atol=1e-5						)						)

								# The output should be different for long inputs
								self.assertFalse(torch.allclose(lowercase_      ,							lowercase_      ,							atol=1e-5						)						)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase           =					{'''configuration_xlnet''': ['''XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLNetConfig''']}

try:
       if not is_sentencepiece_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase           =					['''XLNetTokenizer''']

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase           =					['''XLNetTokenizerFast''']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase           =					[
           '''XLNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''XLNetForMultipleChoice''',
           '''XLNetForQuestionAnswering''',
           '''XLNetForQuestionAnsweringSimple''',
           '''XLNetForSequenceClassification''',
           '''XLNetForTokenClassification''',
           '''XLNetLMHeadModel''',
           '''XLNetModel''',
           '''XLNetPreTrainedModel''',
           '''load_tf_weights_in_xlnet''',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase           =					[
           '''TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''TFXLNetForMultipleChoice''',
           '''TFXLNetForQuestionAnsweringSimple''',
           '''TFXLNetForSequenceClassification''',
           '''TFXLNetForTokenClassification''',
           '''TFXLNetLMHeadModel''',
           '''TFXLNetMainLayer''',
           '''TFXLNetModel''',
           '''TFXLNetPreTrainedModel''',
       ]


if TYPE_CHECKING:
       from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

       try:
              if not is_sentencepiece_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_xlnet import XLNetTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_xlnet_fast import XLNetTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_xlnet import (
                  XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XLNetForMultipleChoice,
                  XLNetForQuestionAnswering,
                  XLNetForQuestionAnsweringSimple,
                  XLNetForSequenceClassification,
                  XLNetForTokenClassification,
                  XLNetLMHeadModel,
                  XLNetModel,
                  XLNetPreTrainedModel,
                  load_tf_weights_in_xlnet,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_xlnet import (
                  TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFXLNetForMultipleChoice,
                  TFXLNetForQuestionAnsweringSimple,
                  TFXLNetForSequenceClassification,
                  TFXLNetForTokenClassification,
                  TFXLNetLMHeadModel,
                  TFXLNetMainLayer,
                  TFXLNetModel,
                  TFXLNetPreTrainedModel,
              )

else:
       import sys

       lowerCamelCase           =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
