




'''simple docstring'''





import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    FalconForCausalLM,
		    FalconForQuestionAnswering,
		    FalconForSequenceClassification,
		    FalconForTokenClassification,
		    FalconModel,
		)




class  _snake_case		:




			def __init__(					self				,							a__				,							a__=3				,							a__=7				,							a__=True				,							a__=True				,							a__=False				,							a__=True				,							a__=99				,							a__=32				,							a__=5				,							a__=4				,							a__=37				,							a__="gelu"				,							a__=0.1				,							a__=0.1				,							a__=512				,							a__=16				,							a__=2				,							a__=0.0_2				,							a__=3				,							a__=4				,							a__=None				,							)				->				List[str]:

										'''simple docstring'''




										snake_case_						=      parent
										snake_case_						=      batch_size
										snake_case_						=      seq_length
										snake_case_						=      is_training
										snake_case_						=      use_input_mask
										snake_case_						=      use_token_type_ids
										snake_case_						=      use_labels
										snake_case_						=      vocab_size
										snake_case_						=      hidden_size
										snake_case_						=      num_hidden_layers
										snake_case_						=      num_attention_heads
										snake_case_						=      intermediate_size
										snake_case_						=      hidden_act
										snake_case_						=      hidden_dropout_prob
										snake_case_						=      attention_probs_dropout_prob
										snake_case_						=      max_position_embeddings
										snake_case_						=      type_vocab_size
										snake_case_						=      type_sequence_label_size
										snake_case_						=      initializer_range
										snake_case_						=      num_labels
										snake_case_						=      num_choices
										snake_case_						=      scope




			def    lowerCAmelCase__				(					self							)				->				int:

										'''simple docstring'''




										snake_case_						=      ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size							)

										snake_case_						=      None
										if self.use_input_mask:
																	snake_case_						=      random_attention_mask([self.batch_size, self.seq_length]							)

										snake_case_						=      None

										snake_case_						=      None
										snake_case_						=      None
										snake_case_						=      None
										if self.use_labels:
																	snake_case_						=      ids_tensor([self.batch_size]				,							self.type_sequence_label_size							)
																	snake_case_						=      ids_tensor([self.batch_size, self.seq_length]				,							self.num_labels							)
																	snake_case_						=      ids_tensor([self.batch_size]				,							self.num_choices							)

										snake_case_						=      self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




			def    lowerCAmelCase__				(					self							)				->				Optional[Any]:

										'''simple docstring'''




										return FalconConfig(
										    vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							num_hidden_layers=self.num_hidden_layers				,							num_attention_heads=self.num_attention_heads				,							intermediate_size=self.intermediate_size				,							hidden_act=self.hidden_act				,							hidden_dropout_prob=self.hidden_dropout_prob				,							attention_probs_dropout_prob=self.attention_probs_dropout_prob				,							max_position_embeddings=self.max_position_embeddings				,							type_vocab_size=self.type_vocab_size				,							is_decoder=a__				,							initializer_range=self.initializer_range				,							pad_token_id=1				,							new_decoder_architecture=a__				,							)




			def    lowerCAmelCase__				(					self				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__							)				->				Optional[Any]:

										'''simple docstring'''




										snake_case_						=      FalconModel(config=a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(a__				,							attention_mask=a__							)
										snake_case_						=      model(a__							)
										self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)							)




			def    lowerCAmelCase__				(					self				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							)				->				Optional[int]:

										'''simple docstring'''




										snake_case_						=      True
										snake_case_						=      FalconModel(a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(
										    a__				,							attention_mask=a__				,							encoder_hidden_states=a__				,							encoder_attention_mask=a__				,							)
										snake_case_						=      model(
										    a__				,							attention_mask=a__				,							encoder_hidden_states=a__				,							)
										snake_case_						=      model(a__				,							attention_mask=a__							)
										self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)							)




			def    lowerCAmelCase__				(					self				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							)				->				str:

										'''simple docstring'''




										snake_case_						=      FalconForCausalLM(config=a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(a__				,							attention_mask=a__				,							labels=a__							)
										self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)							)




			def    lowerCAmelCase__				(					self				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							a__				,							)				->				Dict:

										'''simple docstring'''




										snake_case_						=      True
										snake_case_						=      True
										snake_case_						=      FalconForCausalLM(config=a__							)
										model.to(a__							)
										model.eval()

										# first forward pass
										snake_case_						=      model(
										    a__				,							attention_mask=a__				,							encoder_hidden_states=a__				,							encoder_attention_mask=a__				,							use_cache=a__				,							)
										snake_case_						=      outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										snake_case_						=      ids_tensor((self.batch_size, 3)				,							config.vocab_size							)
										snake_case_						=      ids_tensor((self.batch_size, 3)				,							vocab_size=2							)

										# append to next input_ids and
										snake_case_						=      torch.cat([input_ids, next_tokens]				,							dim=-1							)
										snake_case_						=      torch.cat([input_mask, next_mask]				,							dim=-1							)

										snake_case_						=      model(
										    a__				,							attention_mask=a__				,							encoder_hidden_states=a__				,							encoder_attention_mask=a__				,							output_hidden_states=a__				,							)["hidden_states"][0]
										snake_case_						=      model(
										    a__				,							attention_mask=a__				,							encoder_hidden_states=a__				,							encoder_attention_mask=a__				,							past_key_values=a__				,							output_hidden_states=a__				,							)["hidden_states"][0]

										# select random slice
										snake_case_						=      ids_tensor((1,)				,							output_from_past.shape[-1]							).item()
										snake_case_						=      output_from_no_past[:, -3:, random_slice_idx].detach()
										snake_case_						=      output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(a__				,							a__				,							atol=1e-3							)							)




			def    lowerCAmelCase__				(					self							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_						=      self.prepare_config_and_inputs()
										(
										    (
										    snake_case_
										)					,      (
										    snake_case_
										)					,      (
										    snake_case_
										)					,      (
										    snake_case_
										)					,      (
										    snake_case_
										)					,      (
										    snake_case_
										)					,      (
										    snake_case_
										)					,      
										)						=      config_and_inputs
										snake_case_						=      {"input_ids": input_ids, "attention_mask": input_mask}
										return config, inputs_dict




@require_torch
class  _snake_case		(	lowercase_						,						lowercase_						,						lowercase_						,						unittest.TestCase							):
			lowerCAmelCase_     :      Optional[int]						       =    (
			    (
			        FalconModel,
			        FalconForCausalLM,
			        FalconForSequenceClassification,
			        FalconForTokenClassification,
			        FalconForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCAmelCase_     :      List[Any]						       =    (FalconForCausalLM,) if is_torch_available() else ()
			lowerCAmelCase_     :      Union[str, Any]						       =    (
			    {
			        "feature-extraction": FalconModel,
			        "text-classification": FalconForSequenceClassification,
			        "text-generation": FalconForCausalLM,
			        "question-answering": FalconForQuestionAnswering,
			        "token-classification": FalconForTokenClassification,
			        "zero-shot": FalconForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCAmelCase_     :      Optional[Any]						       =    False
			lowerCAmelCase_     :      List[Any]						       =    False




			def    lowerCAmelCase__				(					self							)				->				int:

										'''simple docstring'''




										snake_case_						=      FalconModelTester(self							)
										snake_case_						=      ConfigTester(self				,							config_class=a__				,							hidden_size=37							)




			def    lowerCAmelCase__				(					self							)				->				Union[str, Any]:

										'''simple docstring'''




										self.config_tester.run_common_tests()




			def    lowerCAmelCase__				(					self							)				->				Optional[int]:

										'''simple docstring'''




										snake_case_						=      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*a__							)




			def    lowerCAmelCase__				(					self							)				->				List[str]:

										'''simple docstring'''




										snake_case_					,      *snake_case_						=      self.model_tester.prepare_config_and_inputs()
										for alibi in [True, False]:
																	snake_case_						=      alibi
																	self.model_tester.create_and_check_model(a__				,							*a__							)




			def    lowerCAmelCase__				(					self							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()
										snake_case_						=      3
										snake_case_						=      input_dict["input_ids"]
										snake_case_						=      input_ids.ne(1							).to(a__							)
										snake_case_						=      ids_tensor([self.model_tester.batch_size]				,							self.model_tester.type_sequence_label_size							)
										snake_case_						=      FalconForSequenceClassification(a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(a__				,							attention_mask=a__				,							labels=a__							)
										self.assertEqual(result.logits.shape				,							(self.model_tester.batch_size, self.model_tester.num_labels)							)




			def    lowerCAmelCase__				(					self							)				->				List[str]:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()
										snake_case_						=      3
										snake_case_						=      "single_label_classification"
										snake_case_						=      input_dict["input_ids"]
										snake_case_						=      input_ids.ne(1							).to(a__							)
										snake_case_						=      ids_tensor([self.model_tester.batch_size]				,							self.model_tester.type_sequence_label_size							)
										snake_case_						=      FalconForSequenceClassification(a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(a__				,							attention_mask=a__				,							labels=a__							)
										self.assertEqual(result.logits.shape				,							(self.model_tester.batch_size, self.model_tester.num_labels)							)




			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()
										snake_case_						=      input_dict["input_ids"]
										snake_case_						=      FalconForCausalLM(a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(a__				,							use_cache=a__							)
										snake_case_						=      input_ids.shape[0]
										snake_case_						=      model._convert_to_rw_cache(result.past_key_values							)
										snake_case_						=      model._convert_cache_to_standard_format(a__				,							a__							)
										for layer in range(len(a__							)							):
																	for tensor_idx in range(2							):
																								self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3							)
																								self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4							)
																								self.assertTrue(
																								    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]							)							)




			def    lowerCAmelCase__				(					self							)				->				Union[str, Any]:

										'''simple docstring'''




										snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()
										snake_case_						=      3
										snake_case_						=      "multi_label_classification"
										snake_case_						=      input_dict["input_ids"]
										snake_case_						=      input_ids.ne(1							).to(a__							)
										snake_case_						=      ids_tensor(
										    [self.model_tester.batch_size, config.num_labels]				,							self.model_tester.type_sequence_label_size							).to(torch.float							)
										snake_case_						=      FalconForSequenceClassification(a__							)
										model.to(a__							)
										model.eval()
										snake_case_						=      model(a__				,							attention_mask=a__				,							labels=a__							)
										self.assertEqual(result.logits.shape				,							(self.model_tester.batch_size, self.model_tester.num_labels)							)




			def    lowerCAmelCase__				(					self							)				->				str:

										'''simple docstring'''




										for model_class in self.all_generative_model_classes:
																	snake_case_					,      snake_case_						=      self.model_tester.prepare_config_and_inputs_for_common()

																	# If it doesn't support cache, pass the test
																	if not hasattr(a__				,							"use_cache"							):
																								return

																	snake_case_						=      model_class(a__							).to(a__							)
																	if "use_cache" not in inputs:
																								snake_case_						=      True
																	snake_case_						=      model(**a__							)

																	# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
																	if "past_key_values" not in outputs:
																								return

																	snake_case_						=      (
																	    getattr(a__				,							"decoder_layers"				,							a__							)
																	    or getattr(a__				,							"num_decoder_layers"				,							a__							)
																	    or config.num_hidden_layers
																	)
																	snake_case_						=      getattr(a__				,							"num_kv_heads"				,							config.num_attention_heads							)
																	snake_case_						=      getattr(a__				,							"d_model"				,							config.hidden_size							)
																	snake_case_						=      embed_dim // num_attention_heads

																	snake_case_						=      outputs["past_key_values"]
																	self.assertEqual(len(a__							)				,							a__							)

																	snake_case_					,      snake_case_						=      inputs["input_ids"].shape
																	for i in range(a__							):
																								if config.new_decoder_architecture:
																															snake_case_						=      config.num_attention_heads
																								elif config.multi_query:
																															snake_case_						=      1
																								self.assertEqual(len(past_kv[0]							)				,							2							)  # K V for the decoder = 2
																								self.assertEqual(
																								    past_kv[i][0].shape				,							(batch_size, num_attention_heads, seq_length, per_head_embed_dim)							)
																								self.assertEqual(
																								    past_kv[i][1].shape				,							(batch_size, num_attention_heads, seq_length, per_head_embed_dim)							)




@require_torch
class  _snake_case		(	unittest.TestCase							):




			@slow
			def    lowerCAmelCase__				(					self							)				->				Any:

										'''simple docstring'''




										snake_case_						=      AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b"							)
										snake_case_						=      FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b"							)
										model.eval()
										model.to(a__							)
										snake_case_						=      tokenizer("My favorite food is"				,							return_tensors="pt"							).to(a__							)

										snake_case_						=      (
										    "My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."
										)

										snake_case_						=      model.generate(**a__				,							do_sample=a__				,							max_new_tokens=19							)
										snake_case_						=      tokenizer.batch_decode(a__							)[0]

										self.assertEqual(a__				,							a__							)




			@slow
			def    lowerCAmelCase__				(					self							)				->				Any:

										'''simple docstring'''




										for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
																	snake_case_						=      AutoTokenizer.from_pretrained(a__							)
																	snake_case_						=      FalconForCausalLM.from_pretrained(a__							)
																	model.eval()
																	model.to(a__							)
																	snake_case_						=      tokenizer("My favorite food is"				,							return_tensors="pt"							).to(a__							)

																	# We just test that these run without errors - the models are randomly initialized
																	# and so the actual text outputs will be garbage
																	model.generate(**a__				,							do_sample=a__				,							max_new_tokens=4							)
																	model.generate(**a__				,							do_sample=a__				,							max_new_tokens=4							)
																	model.generate(**a__				,							num_beams=2				,							max_new_tokens=4							)




			@slow
			def    lowerCAmelCase__				(					self							)				->				int:

										'''simple docstring'''




										with torch.no_grad():
																	for repo in [
																	    "Rocketknight1/falcon-rw-1b",
																	    "Rocketknight1/tiny-random-falcon-7b",
																	    "Rocketknight1/tiny-random-falcon-40b",
																	]:
																								snake_case_						=      AutoTokenizer.from_pretrained(a__							)
																								snake_case_						=      FalconForCausalLM.from_pretrained(a__							)
																								model.eval()
																								model.to(device=a__							)
																								snake_case_						=      tokenizer("My favorite food is"				,							return_tensors="pt"							).to(a__							)

																								# Test results are the same with and without cache
																								snake_case_						=      model.generate(**a__				,							do_sample=a__				,							max_new_tokens=20				,							use_cache=a__							)
																								snake_case_						=      model.generate(**a__				,							do_sample=a__				,							max_new_tokens=20				,							use_cache=a__							)
																								self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0							)






import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase	      = logging.get_logger(__name__)

UpperCAmelCase	      = {
    '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/config.json''',
    '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/config.json''',
}







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	_UpperCamelCase     :     Tuple							 =       """xlnet"""
	_UpperCamelCase     :     Optional[Any]							 =       ["""mems"""]
	_UpperCamelCase     :     Tuple							 =       {
	    """n_token""": """vocab_size""",  # Backward compatibility
	    """hidden_size""": """d_model""",
	    """num_attention_heads""": """n_head""",
	    """num_hidden_layers""": """n_layer""",
	}





	def __init__(       self		,    snake_case=3_2000		,    snake_case=1024		,    snake_case=24		,    snake_case=16		,    snake_case=4096		,    snake_case="gelu"		,    snake_case=True		,    snake_case="bi"		,    snake_case=0.02		,    snake_case=1E-12		,    snake_case=0.1		,    snake_case=512		,    snake_case=None		,    snake_case=True		,    snake_case=False		,    snake_case=False		,    snake_case=-1		,    snake_case=False		,    snake_case="last"		,    snake_case=True		,    snake_case="tanh"		,    snake_case=0.1		,    snake_case=5		,    snake_case=5		,    snake_case=5		,    snake_case=1		,    snake_case=2		,    **snake_case		,    ):
		lowercase       =    vocab_size
		lowercase       =    d_model
		lowercase       =    n_layer
		lowercase       =    n_head
		if d_model % n_head != 0:
			raise ValueError(F'''\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'''       )
		if "d_head" in kwargs:
			if kwargs["d_head"] != d_model // n_head:
				raise ValueError(
				    F'''`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})'''       )
		lowercase       =    d_model // n_head
		lowercase       =    ff_activation
		lowercase       =    d_inner
		lowercase       =    untie_r
		lowercase       =    attn_type

		lowercase       =    initializer_range
		lowercase       =    layer_norm_eps

		lowercase       =    dropout
		lowercase       =    mem_len
		lowercase       =    reuse_len
		lowercase       =    bi_data
		lowercase       =    clamp_len
		lowercase       =    same_length

		lowercase       =    summary_type
		lowercase       =    summary_use_proj
		lowercase       =    summary_activation
		lowercase       =    summary_last_dropout
		lowercase       =    start_n_top
		lowercase       =    end_n_top

		lowercase       =    bos_token_id
		lowercase       =    pad_token_id
		lowercase       =    eos_token_id

		if "use_cache" in kwargs:
			warnings.warn(
			    'The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'
			    ' instead.'		,    snake_case		,    )
			lowercase       =    kwargs['use_cache']

		lowercase       =    use_mems_eval
		lowercase       =    use_mems_train
		super().__init__(pad_token_id=snake_case		,    bos_token_id=snake_case		,    eos_token_id=snake_case		,    **snake_case       )





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''       )
		return -1





	@max_position_embeddings.setter
	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		# Message copied from Transformer-XL documentation
		raise NotImplementedError(
		    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''       )

