



'''simple docstring'''




from pickle import UnpicklingError

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict

from ..utils import logging


lowerCamelCase							:List[str]				    =					logging.get_logger(__name__)



def 	a			( lowerCamelCase__						,      lowerCamelCase__    ):
			'''simple docstring'''





			try:
						with open(lowerCamelCase__						,      """rb"""    ) as flax_state_f:
									A_	:						Optional[Any]	=     from_bytes(lowerCamelCase__						,      flax_state_f.read()    )
			except UnpicklingError as e:
						try:
									with open(lowerCamelCase__    ) as f:
												if f.read().startswith("""version"""    ):
															raise OSError(
															    """You seem to have cloned a repository without having git-lfs installed. Please"""
															    """ install git-lfs and run `git lfs install` followed by `git lfs pull` in the"""
															    """ folder you cloned."""    )
												else:
															raise ValueError from e
						except (UnicodeDecodeError, ValueError):
									raise EnvironmentError(f'Unable to convert {model_file} to Flax deserializable object. '    )

			return load_flax_weights_in_pytorch_model(lowerCamelCase__						,      lowerCamelCase__    )



def 	a			( lowerCamelCase__						,      lowerCamelCase__    ):
			'''simple docstring'''





			try:
						import torch  # noqa: F401
			except ImportError:
						logger.error(
						    """Loading Flax weights in PyTorch requires both PyTorch and Flax to be installed. Please see"""
						    """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
						    """ instructions."""    )
						raise

			# check if we have bf16 weights
			A_	:						List[str]	=     flatten_dict(jax.tree_util.tree_map(lambda lowerCamelCase__    : x.dtype == jnp.bfloataa						,      lowerCamelCase__    )    ).values()
			if any(lowerCamelCase__    ):
						# convert all weights to fp32 if they are bf16 since torch.from_numpy can-not handle bf16

						# and bf16 is not fully supported in PT yet.
						logger.warning(
						    """Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` """
						    """before loading those in PyTorch model."""    )
						A_	:						Dict	=     jax.tree_util.tree_map(
						    lambda lowerCamelCase__    : params.astype(np.floataa    ) if params.dtype == jnp.bfloataa else params						,      lowerCamelCase__    )

			A_	:						Optional[int]	=     """"""

			A_	:						Dict	=     flatten_dict(lowerCamelCase__						,      sep="""."""    )
			A_	:						List[Any]	=     pt_model.state_dict()

			# keep track of unexpected & missing keys
			A_	:						List[str]	=     []
			A_	:						int	=     set(pt_model_dict.keys()    )

			for flax_key_tuple, flax_tensor in flax_state_dict.items():
						A_	:						Tuple	=     flax_key_tuple.split("""."""    )

						if flax_key_tuple_array[-1] == "kernel" and flax_tensor.ndim == 4:
									A_	:						List[Any]	=     flax_key_tuple_array[:-1] + ["""weight"""]
									A_	:						str	=     jnp.transpose(lowerCamelCase__						,      (3, 2, 0, 1)    )
						elif flax_key_tuple_array[-1] == "kernel":
									A_	:						Dict	=     flax_key_tuple_array[:-1] + ["""weight"""]
									A_	:						int	=     flax_tensor.T
						elif flax_key_tuple_array[-1] == "scale":
									A_	:						Dict	=     flax_key_tuple_array[:-1] + ["""weight"""]

						if "time_embedding" not in flax_key_tuple_array:
									for i, flax_key_tuple_string in enumerate(lowerCamelCase__    ):
												A_	:						Union[str, Any]	=     (
												    flax_key_tuple_string.replace("""_0"""						,      """.0"""    )
												    .replace("""_1"""						,      """.1"""    )
												    .replace("""_2"""						,      """.2"""    )
												    .replace("""_3"""						,      """.3"""    )
												    .replace("""_4"""						,      """.4"""    )
												    .replace("""_5"""						,      """.5"""    )
												    .replace("""_6"""						,      """.6"""    )
												    .replace("""_7"""						,      """.7"""    )
												    .replace("""_8"""						,      """.8"""    )
												    .replace("""_9"""						,      """.9"""    )
												)

						A_	:						Any	=     """.""".join(lowerCamelCase__    )

						if flax_key in pt_model_dict:
									if flax_tensor.shape != pt_model_dict[flax_key].shape:
												raise ValueError(
												    f'Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '
												    f'to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'    )
									else:
												# add weight to pytorch dict
												A_	:						Optional[int]	=     np.asarray(lowerCamelCase__    ) if not isinstance(lowerCamelCase__						,      np.ndarray    ) else flax_tensor
												A_	:						List[str]	=     torch.from_numpy(lowerCamelCase__    )
												# remove from missing keys
												missing_keys.remove(lowerCamelCase__    )
						else:
									# weight is not expected by PyTorch model
									unexpected_keys.append(lowerCamelCase__    )

			pt_model.load_state_dict(lowerCamelCase__    )

			# re-transform missing_keys to list
			A_	:						Any	=     list(lowerCamelCase__    )

			if len(lowerCamelCase__    ) > 0:
						logger.warning(
						    """Some weights of the Flax model were not used when initializing the PyTorch model"""
						    f' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'
						    f' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'
						    """ (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This"""
						    f' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'
						    """ to be exactly identical (e.g. initializing a BertForSequenceClassification model from a"""
						    """ FlaxBertForSequenceClassification model)."""    )
			if len(lowerCamelCase__    ) > 0:
						logger.warning(
						    f'Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'
						    f' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'
						    """ use it for predictions and inference."""    )

			return pt_model




'''simple docstring'''




import unittest

import numpy as np

from transformers import DistilBertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
						import jax.numpy as jnp

						from transformers.models.distilbert.modeling_flax_distilbert import (
						    FlaxDistilBertForMaskedLM,
						    FlaxDistilBertForMultipleChoice,
						    FlaxDistilBertForQuestionAnswering,
						    FlaxDistilBertForSequenceClassification,
						    FlaxDistilBertForTokenClassification,
						    FlaxDistilBertModel,
						)







class 	_lowerCAmelCase							(			unittest.TestCase ):


	def __init__(self       ,      lowercase       ,      lowercase=13       ,      lowercase=7       ,      lowercase=True       ,      lowercase=True       ,      lowercase=True       ,      lowercase=True       ,      lowercase=99       ,      lowercase=32       ,      lowercase=5       ,      lowercase=4       ,      lowercase=37       ,      lowercase="gelu"       ,      lowercase=0.1       ,      lowercase=0.1       ,      lowercase=512       ,      lowercase=16       ,      lowercase=2       ,      lowercase=0.02       ,      lowercase=4       ,      ):
				A_	:						List[Any]	=     parent
				A_	:						str	=     batch_size
				A_	:						List[Any]	=     seq_length
				A_	:						Dict	=     is_training
				A_	:						List[Any]	=     use_attention_mask
				A_	:						Any	=     use_token_type_ids
				A_	:						Optional[int]	=     use_labels
				A_	:						Tuple	=     vocab_size
				A_	:						List[str]	=     hidden_size
				A_	:						List[str]	=     num_hidden_layers
				A_	:						Optional[Any]	=     num_attention_heads
				A_	:						int	=     intermediate_size
				A_	:						Optional[Any]	=     hidden_act
				A_	:						List[Any]	=     hidden_dropout_prob
				A_	:						Optional[Any]	=     attention_probs_dropout_prob
				A_	:						Any	=     max_position_embeddings
				A_	:						Union[str, Any]	=     type_vocab_size
				A_	:						int	=     type_sequence_label_size
				A_	:						Any	=     initializer_range
				A_	:						List[str]	=     num_choices


	def     _a	(self							):
				A_	:						List[str]	=     ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size							)

				A_	:						Any	=     None
				if self.use_attention_mask:
							A_	:						Any	=     random_attention_mask([self.batch_size, self.seq_length]							)

				A_	:						Union[str, Any]	=     DistilBertConfig(
				    vocab_size=self.vocab_size       ,      dim=self.hidden_size       ,      n_layers=self.num_hidden_layers       ,      n_heads=self.num_attention_heads       ,      hidden_dim=self.intermediate_size       ,      hidden_act=self.hidden_act       ,      dropout=self.hidden_dropout_prob       ,      attention_dropout=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      initializer_range=self.initializer_range       ,      tie_weights_=lowercase       ,      )

				return config, input_ids, attention_mask


	def     _a	(self							):
				A_	:						List[str]	=     self.prepare_config_and_inputs()
				A_,    A_,    A_	:						str	=     config_and_inputs
				A_	:						Any	=     {"""input_ids""": input_ids, """attention_mask""": attention_mask}
				return config, inputs_dict







@require_flax
class 	_lowerCAmelCase							(			__UpperCAmelCase      ,			unittest.TestCase ):
	__SCREAMING_SNAKE_CASE   :		Optional[Any]					=					(
	    (
	        FlaxDistilBertModel,
	        FlaxDistilBertForMaskedLM,
	        FlaxDistilBertForMultipleChoice,
	        FlaxDistilBertForQuestionAnswering,
	        FlaxDistilBertForSequenceClassification,
	        FlaxDistilBertForTokenClassification,
	        FlaxDistilBertForQuestionAnswering,
	    )
	    if is_flax_available()
	    else ()
	)


	def     _a	(self							):
				A_	:						Tuple	=     FlaxDistilBertModelTester(self							)


	@slow
	def     _a	(self							):
				for model_class_name in self.all_model_classes:
							A_	:						Union[str, Any]	=     model_class_name.from_pretrained("""distilbert-base-uncased"""							)
							A_	:						Any	=     model(np.ones((1, 1)							)							)
							self.assertIsNotNone(lowercase							)







@require_flax
class 	_lowerCAmelCase							(			unittest.TestCase ):


	@slow
	def     _a	(self							):
				A_	:						List[str]	=     FlaxDistilBertModel.from_pretrained("""distilbert-base-uncased"""							)
				A_	:						Optional[Any]	=     np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]							)
				A_	:						int	=     np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
				A_	:						Optional[int]	=     model(lowercase       ,      attention_mask=lowercase							)[0]
				A_	:						Optional[Any]	=     (1, 11, 768)
				self.assertEqual(output.shape       ,      lowercase							)
				A_	:						Union[str, Any]	=     np.array([[[-0.16_39, 0.32_99, 0.16_48], [-0.17_46, 0.32_89, 0.17_10], [-0.18_84, 0.33_57, 0.18_10]]]							)

				self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]       ,      lowercase       ,      atol=1E-4							)							)