

import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
			from transformers.models.esm.modeling_esm import (
			    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
			    EsmEmbeddings,
			    create_position_ids_from_input_ids,
			)

class 				UpperCAmelCase_		:
							"""simple docstring"""







							def __init__(			self       ,							_a       ,							_a=1_3       ,							_a=7       ,							_a=False       ,							_a=True       ,							_a=False       ,							_a=True       ,							_a=3_3       ,							_a=3_2       ,							_a=5       ,							_a=4       ,							_a=3_7       ,							_a="gelu"       ,							_a=0.1       ,							_a=0.1       ,							_a=5_1_2       ,							_a=1_6       ,							_a=2       ,							_a=0.02       ,							_a=3       ,							_a=4       ,							_a=None       ,							)    ->    List[str]:
												_a	:    List[str]         =		parent
												_a	:    Optional[Any]         =		batch_size
												_a	:    List[Any]         =		seq_length
												_a	:    Optional[int]         =		is_training
												_a	:    str         =		use_input_mask
												_a	:    Tuple         =		use_token_type_ids
												_a	:    Optional[Any]         =		use_labels
												_a	:    Optional[int]         =		vocab_size
												_a	:    Optional[Any]         =		hidden_size
												_a	:    List[str]         =		num_hidden_layers
												_a	:    str         =		num_attention_heads
												_a	:    List[str]         =		intermediate_size
												_a	:    Optional[int]         =		hidden_act
												_a	:    Tuple         =		hidden_dropout_prob
												_a	:    Optional[Any]         =		attention_probs_dropout_prob
												_a	:    str         =		max_position_embeddings
												_a	:    Optional[Any]         =		type_vocab_size
												_a	:    Any         =		type_sequence_label_size
												_a	:    Any         =		initializer_range
												_a	:    Optional[int]         =		num_labels
												_a	:    Optional[int]         =		num_choices
												_a	:    str         =		scope


							def 				__lowercase				(			self       )    ->    Union[str, Any]:
												_a	:    str         =		ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size       )

												_a	:    Union[str, Any]         =		None
												if self.use_input_mask:
																	_a	:    str         =		random_attention_mask([self.batch_size, self.seq_length]       )

												_a	:    str         =		None
												_a	:    Optional[Any]         =		None
												_a	:    Dict         =		None
												if self.use_labels:
																	_a	:    List[str]         =		ids_tensor([self.batch_size]       ,							self.type_sequence_label_size       )
																	_a	:    Tuple         =		ids_tensor([self.batch_size, self.seq_length]       ,							self.num_labels       )
																	_a	:    List[Any]         =		ids_tensor([self.batch_size]       ,							self.num_choices       )

												_a	:    List[Any]         =		self.get_config()

												return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


							def 				__lowercase				(			self       )    ->    Optional[Any]:
												return EsmConfig(
												    vocab_size=self.vocab_size       ,							hidden_size=self.hidden_size       ,							pad_token_id=1       ,							num_hidden_layers=self.num_hidden_layers       ,							num_attention_heads=self.num_attention_heads       ,							intermediate_size=self.intermediate_size       ,							hidden_act=self.hidden_act       ,							hidden_dropout_prob=self.hidden_dropout_prob       ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,							max_position_embeddings=self.max_position_embeddings       ,							type_vocab_size=self.type_vocab_size       ,							initializer_range=self.initializer_range       ,							)


							def 				__lowercase				(			self       ,							_a       ,							_a       ,							_a       ,							_a       ,							_a       ,							_a       )    ->    Optional[Any]:
												_a	:    List[Any]         =		EsmModel(config=_a       )
												model.to(_a       )
												model.eval()
												_a	:    Tuple         =		model(_a       ,							attention_mask=_a       )
												_a	:    Optional[int]         =		model(_a       )
												_a	:    Optional[Any]         =		model(_a       )

												self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)       )
												self.parent.assertEqual(result.pooler_output.shape       ,							(self.batch_size, self.hidden_size)       )


							def 				__lowercase				(			self       ,							_a       ,							_a       ,							_a       ,							_a       ,							_a       ,							_a       )    ->    Tuple:
												_a	:    List[Any]         =		EsmForMaskedLM(config=_a       )
												model.to(_a       )
												model.eval()
												_a	:    List[Any]         =		model(_a       ,							attention_mask=_a       ,							labels=_a       )
												self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.vocab_size)       )


							def 				__lowercase				(			self       ,							_a       ,							_a       ,							_a       ,							_a       ,							_a       ,							_a       )    ->    Optional[Any]:
												_a	:    Any         =		self.num_labels
												_a	:    List[str]         =		EsmForTokenClassification(config=_a       )
												model.to(_a       )
												model.eval()
												_a	:    Optional[int]         =		model(_a       ,							attention_mask=_a       ,							labels=_a       )
												self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.num_labels)       )




							def 				__lowercase				(			self       )    ->    int:
												_a	:    Union[str, Any]         =		self.prepare_config_and_inputs()
												(
												    (
												    _a
												)      ,					(
												    _a
												)      ,					(
												    _a
												)      ,					(
												    _a
												)      ,					(
												    _a
												)      ,					(
												    _a
												)      ,					
												)	:    Dict         =		config_and_inputs
												_a	:    Tuple         =		{'''input_ids''': input_ids, '''attention_mask''': input_mask}
												return config, inputs_dict



@require_torch
class 				UpperCAmelCase_		(			__lowercase				,       __lowercase				,       unittest.TestCase   ):
							"""simple docstring"""







							UpperCAmelCase__       :						List[Any]	    =    False

							UpperCAmelCase__       :						int	    =    (
							    (
							        EsmForMaskedLM,
							        EsmModel,
							        EsmForSequenceClassification,
							        EsmForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							UpperCAmelCase__       :						int	    =    ()
							UpperCAmelCase__       :						Any	    =    (
							    {
							        "feature-extraction": EsmModel,
							        "fill-mask": EsmForMaskedLM,
							        "text-classification": EsmForSequenceClassification,
							        "token-classification": EsmForTokenClassification,
							        "zero-shot": EsmForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							UpperCAmelCase__       :						List[str]	    =    True


							def 				__lowercase				(			self       )    ->    Tuple:
												_a	:    str         =		EsmModelTester(self       )
												_a	:    Union[str, Any]         =		ConfigTester(self       ,							config_class=_a       ,							hidden_size=3_7       )


							def 				__lowercase				(			self       )    ->    str:
												self.config_tester.run_common_tests()


							def 				__lowercase				(			self       )    ->    Dict:
												_a	:    Optional[Any]         =		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*_a       )


							def 				__lowercase				(			self       )    ->    List[str]:
												_a	:    List[str]         =		self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																	_a	:    Tuple         =		type
																	self.model_tester.create_and_check_model(*_a       )


							def 				__lowercase				(			self       )    ->    Tuple:
												_a	:    Dict         =		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*_a       )


							def 				__lowercase				(			self       )    ->    List[Any]:
												_a	:    int         =		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*_a       )


							@slow
							def 				__lowercase				(			self       )    ->    Optional[Any]:
												for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	_a	:    List[Any]         =		EsmModel.from_pretrained(_a       )
																	self.assertIsNotNone(_a       )


							def 				__lowercase				(			self       )    ->    Any:
												_a	:    List[Any]         =		self.model_tester.prepare_config_and_inputs()[0]
												_a	:    Dict         =		EsmEmbeddings(config=_a       )

												_a	:    Optional[Any]         =		torch.as_tensor([[1_2, 3_1, 1_3, model.padding_idx]]       )
												_a	:    Optional[int]         =		torch.as_tensor(
												    [
												        [
												            0 + model.padding_idx + 1,
												            1 + model.padding_idx + 1,
												            2 + model.padding_idx + 1,
												            model.padding_idx,
												        ]
												    ]       )
												_a	:    List[Any]         =		create_position_ids_from_input_ids(_a       ,							model.padding_idx       )
												self.assertEqual(position_ids.shape       ,							expected_positions.shape       )
												self.assertTrue(torch.all(torch.eq(_a       ,							_a       )       )       )


							def 				__lowercase				(			self       )    ->    Optional[Any]:
												_a	:    Tuple         =		self.model_tester.prepare_config_and_inputs()[0]
												_a	:    List[Any]         =		EsmEmbeddings(config=_a       )

												_a	:    List[Any]         =		torch.empty(2       ,							4       ,							3_0       )
												_a	:    List[str]         =		[
												    0 + embeddings.padding_idx + 1,
												    1 + embeddings.padding_idx + 1,
												    2 + embeddings.padding_idx + 1,
												    3 + embeddings.padding_idx + 1,
												]
												_a	:    Optional[Any]         =		torch.as_tensor([expected_single_positions, expected_single_positions]       )
												_a	:    str         =		embeddings.create_position_ids_from_inputs_embeds(_a       )
												self.assertEqual(position_ids.shape       ,							expected_positions.shape       )
												self.assertTrue(torch.all(torch.eq(_a       ,							_a       )       )       )


							@unittest.skip('''Esm does not support embedding resizing'''       )
							def 				__lowercase				(			self       )    ->    int:
												pass


							@unittest.skip('''Esm does not support embedding resizing'''       )
							def 				__lowercase				(			self       )    ->    Union[str, Any]:
												pass




							@unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''       )
							def 				__lowercase				(			self       )    ->    List[Any]:
												pass



@require_torch
class 				UpperCAmelCase_		(			__lowercase   ):
							"""simple docstring"""







							@slow
							def 				__lowercase				(			self       )    ->    Optional[int]:
												with torch.no_grad():
																	_a	:    Optional[Any]         =		EsmForMaskedLM.from_pretrained('''facebook/esm2_t6_8M_UR50D'''       )
																	model.eval()
																	_a	:    List[Any]         =		torch.tensor([[0, 1, 2, 3, 4, 5]]       )
																	_a	:    Union[str, Any]         =		model(_a       )[0]

																	_a	:    Dict         =		3_3

																	_a	:    List[Any]         =		torch.Size((1, 6, vocab_size)       )
																	self.assertEqual(output.shape       ,							_a       )

																	_a	:    Dict         =		torch.tensor(
																	    [[[8.9215, -10.5898, -6.4671], [-6.3967, -13.9114, -1.1212], [-7.7812, -13.9516, -3.7406]]]       )
																	self.assertTrue(torch.allclose(output[:, :3, :3]       ,							_a       ,							atol=1e-4       )       )




							@slow
							def 				__lowercase				(			self       )    ->    int:
												with torch.no_grad():
																	_a	:    Optional[Any]         =		EsmModel.from_pretrained('''facebook/esm2_t6_8M_UR50D'''       )
																	model.eval()

																	_a	:    Union[str, Any]         =		torch.tensor([[0, 6, 4, 1_3, 5, 4, 1_6, 1_2, 1_1, 7, 2]]       )
																	_a	:    str         =		model(_a       )[0]
																	# compare the actual values for a slice.
																	_a	:    Union[str, Any]         =		torch.tensor(
																	    [[[0.1444, 0.5413, 0.3248], [0.3034, 0.0053, 0.3108], [0.3228, -0.2499, 0.3415]]]       )
																	self.assertTrue(torch.allclose(output[:, :3, :3]       ,							_a       ,							atol=1e-4       )       )






import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


a__				=		logging.get_logger(__name__)



def    __UpperCAmelCase (	__a      :			Dict	)			->     Tuple:

					"""simple docstring"""

					_a	:    Any         =		R'''\w+[.]\d+'''
					_a	:    Union[str, Any]         =		re.findall(__a		,__a	)
					for pat in pats:
										_a	:    int         =		key.replace(__a		,'''_'''.join(pat.split('''.'''	)	)	)
					return key



def    __UpperCAmelCase (	__a      :			List[str]		,__a      :			Union[str, Any]		,__a      :			Optional[int]	)			->     Tuple:

					"""simple docstring"""

					_a	:    Dict         =		pt_tuple_key[:-1] + ('''scale''',)
					if (
					    any('''norm''' in str_ for str_ in pt_tuple_key	)
					    and (pt_tuple_key[-1] == "bias")
					    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
					    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
					):
										_a	:    Dict         =		pt_tuple_key[:-1] + ('''scale''',)
										return renamed_pt_tuple_key, pt_tensor
					elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
										_a	:    Optional[int]         =		pt_tuple_key[:-1] + ('''scale''',)
										return renamed_pt_tuple_key, pt_tensor

					# embedding
					if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
										_a	:    Union[str, Any]         =		pt_tuple_key[:-1] + ('''embedding''',)
										return renamed_pt_tuple_key, pt_tensor

					# conv layer
					_a	:    Tuple         =		pt_tuple_key[:-1] + ('''kernel''',)
					if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
										_a	:    List[str]         =		pt_tensor.transpose(2		,3		,1		,0	)
										return renamed_pt_tuple_key, pt_tensor

					# linear layer
					_a	:    Dict         =		pt_tuple_key[:-1] + ('''kernel''',)
					if pt_tuple_key[-1] == "weight":
										_a	:    Union[str, Any]         =		pt_tensor.T
										return renamed_pt_tuple_key, pt_tensor

					# old PyTorch layer norm weight
					_a	:    Dict         =		pt_tuple_key[:-1] + ('''weight''',)
					if pt_tuple_key[-1] == "gamma":
										return renamed_pt_tuple_key, pt_tensor

					# old PyTorch layer norm bias
					_a	:    Union[str, Any]         =		pt_tuple_key[:-1] + ('''bias''',)
					if pt_tuple_key[-1] == "beta":
										return renamed_pt_tuple_key, pt_tensor

					return pt_tuple_key, pt_tensor







def    __UpperCAmelCase (	__a      :			Dict		,__a      :			str		,__a      :			str=42	)			->     Optional[int]:

					"""simple docstring"""

					_a	:    Tuple         =		{k: v.numpy() for k, v in pt_state_dict.items()}

					# Step 2: Since the model is stateless, get random Flax params
					_a	:    List[Any]         =		flax_model.init_weights(PRNGKey(__a	)	)

					_a	:    Optional[int]         =		flatten_dict(__a	)
					_a	:    Tuple         =		{}

					# Need to change some parameters name to match Flax names
					for pt_key, pt_tensor in pt_state_dict.items():
										_a	:    List[str]         =		rename_key(__a	)
										_a	:    Optional[Any]         =		tuple(renamed_pt_key.split('''.'''	)	)

										# Correctly rename weight parameters
										_a      ,					_a	:    List[str]         =		rename_key_and_reshape_tensor(__a		,__a		,__a	)

										if flax_key in random_flax_state_dict:
															if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																				raise ValueError(
																				    F"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
																				    F"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

        # also add unexpected weight so that warning is thrown
										_a	:    Dict         =		jnp.asarray(__a	)

					return unflatten_dict(__a	)



