

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RemBertConfig,
							    RemBertForCausalLM,
							    RemBertForMaskedLM,
							    RemBertForMultipleChoice,
							    RemBertForQuestionAnswering,
							    RemBertForSequenceClassification,
							    RemBertForTokenClassification,
							    RemBertModel,
							)
							from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 						lowercase     :


		"""simple docstring"""




		def __init__(					self						,							__snake_case						,							__snake_case=13						,							__snake_case=7						,							__snake_case=True						,							__snake_case=True						,							__snake_case=True						,							__snake_case=True						,							__snake_case=99						,							__snake_case=32						,							__snake_case=18						,							__snake_case=43						,							__snake_case=5						,							__snake_case=4						,							__snake_case=37						,							__snake_case="gelu"						,							__snake_case=0.1						,							__snake_case=0.1						,							__snake_case=5_12						,							__snake_case=16						,							__snake_case=2						,							__snake_case=0.0_2						,							__snake_case=3						,							__snake_case=4						,							__snake_case=None						,							):
			_UpperCamelCase :		List[str]								=    parent
			_UpperCamelCase :		Union[str, Any]								=    batch_size
			_UpperCamelCase :		int								=    seq_length
			_UpperCamelCase :		Dict								=    is_training
			_UpperCamelCase :		Dict								=    use_input_mask
			_UpperCamelCase :		Union[str, Any]								=    use_token_type_ids
			_UpperCamelCase :		Dict								=    use_labels
			_UpperCamelCase :		int								=    vocab_size
			_UpperCamelCase :		int								=    hidden_size
			_UpperCamelCase :		Dict								=    input_embedding_size
			_UpperCamelCase :		str								=    output_embedding_size
			_UpperCamelCase :		Optional[Any]								=    num_hidden_layers
			_UpperCamelCase :		List[Any]								=    num_attention_heads
			_UpperCamelCase :		Tuple								=    intermediate_size
			_UpperCamelCase :		Dict								=    hidden_act
			_UpperCamelCase :		List[str]								=    hidden_dropout_prob
			_UpperCamelCase :		List[Any]								=    attention_probs_dropout_prob
			_UpperCamelCase :		Tuple								=    max_position_embeddings
			_UpperCamelCase :		int								=    type_vocab_size
			_UpperCamelCase :		Optional[Any]								=    type_sequence_label_size
			_UpperCamelCase :		Any								=    initializer_range
			_UpperCamelCase :		str								=    num_labels
			_UpperCamelCase :		Optional[int]								=    num_choices
			_UpperCamelCase :		str								=    scope


		def 		A__       (					self):
			_UpperCamelCase :		Optional[Any]								=    ids_tensor([self.batch_size, self.seq_length]						,							self.vocab_size)

			_UpperCamelCase :		str								=    None
			if self.use_input_mask:
				_UpperCamelCase :		Any								=    random_attention_mask([self.batch_size, self.seq_length])

			_UpperCamelCase :		str								=    None
			if self.use_token_type_ids:
				_UpperCamelCase :		Any								=    ids_tensor([self.batch_size, self.seq_length]						,							self.type_vocab_size)

			_UpperCamelCase :		Optional[Any]								=    None
			_UpperCamelCase :		List[str]								=    None
			_UpperCamelCase :		int								=    None
			if self.use_labels:
				_UpperCamelCase :		Union[str, Any]								=    ids_tensor([self.batch_size]						,							self.type_sequence_label_size)
				_UpperCamelCase :		int								=    ids_tensor([self.batch_size, self.seq_length]						,							self.num_labels)
				_UpperCamelCase :		Optional[int]								=    ids_tensor([self.batch_size]						,							self.num_choices)

			_UpperCamelCase :		Union[str, Any]								=    RemBertConfig(
			    vocab_size=self.vocab_size						,							hidden_size=self.hidden_size						,							input_embedding_size=self.input_embedding_size						,							output_embedding_size=self.output_embedding_size						,							num_hidden_layers=self.num_hidden_layers						,							num_attention_heads=self.num_attention_heads						,							intermediate_size=self.intermediate_size						,							hidden_act=self.hidden_act						,							hidden_dropout_prob=self.hidden_dropout_prob						,							attention_probs_dropout_prob=self.attention_probs_dropout_prob						,							max_position_embeddings=self.max_position_embeddings						,							type_vocab_size=self.type_vocab_size						,							is_decoder=__snake_case						,							initializer_range=self.initializer_range						,							)

			return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


		def 		A__       (					self):
			(
			    (
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		
			) :		Dict								=    self.prepare_config_and_inputs()

			_UpperCamelCase :		List[Any]								=    True
			_UpperCamelCase :		int								=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
			_UpperCamelCase :		Optional[Any]								=    ids_tensor([self.batch_size, self.seq_length]						,							vocab_size=2)

			return (
			    config,
			    input_ids,
			    token_type_ids,
			    input_mask,
			    sequence_labels,
			    token_labels,
			    choice_labels,
			    encoder_hidden_states,
			    encoder_attention_mask,
			)


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case):
			_UpperCamelCase :		Optional[int]								=    RemBertModel(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		Any								=    model(__snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case)
			_UpperCamelCase :		List[Any]								=    model(__snake_case						,							token_type_ids=__snake_case)
			_UpperCamelCase :		int								=    model(__snake_case)
			self.parent.assertEqual(result.last_hidden_state.shape						,							(self.batch_size, self.seq_length, self.hidden_size))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							):
			_UpperCamelCase :		Dict								=    True
			_UpperCamelCase :		int								=    RemBertModel(__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		Optional[int]								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							encoder_hidden_states=__snake_case						,							encoder_attention_mask=__snake_case						,							)
			_UpperCamelCase :		Optional[int]								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							encoder_hidden_states=__snake_case						,							)
			_UpperCamelCase :		Optional[int]								=    model(__snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case)
			self.parent.assertEqual(result.last_hidden_state.shape						,							(self.batch_size, self.seq_length, self.hidden_size))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							):
			_UpperCamelCase :		Dict								=    RemBertForCausalLM(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		List[str]								=    model(__snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							labels=__snake_case)
			self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.seq_length, self.vocab_size))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case):
			_UpperCamelCase :		Tuple								=    RemBertForMaskedLM(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		List[str]								=    model(__snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							labels=__snake_case)
			self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.seq_length, self.vocab_size))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							):
			_UpperCamelCase :		int								=    True
			_UpperCamelCase :		str								=    True
			_UpperCamelCase :		Optional[Any]								=    RemBertForCausalLM(config=__snake_case)
			model.to(__snake_case)
			model.eval()

			# first forward pass
			_UpperCamelCase :		Tuple								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							encoder_hidden_states=__snake_case						,							encoder_attention_mask=__snake_case						,							use_cache=__snake_case						,							)
			_UpperCamelCase :		Any								=    outputs.past_key_values

			# create hypothetical multiple next token and extent to next_input_ids
			_UpperCamelCase :		Union[str, Any]								=    ids_tensor((self.batch_size, 3)						,							config.vocab_size)
			_UpperCamelCase :		Optional[int]								=    ids_tensor((self.batch_size, 3)						,							vocab_size=2)

			# append to next input_ids and
			_UpperCamelCase :		int								=    torch.cat([input_ids, next_tokens]						,							dim=-1)
			_UpperCamelCase :		str								=    torch.cat([input_mask, next_mask]						,							dim=-1)

			_UpperCamelCase :		Union[str, Any]								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							encoder_hidden_states=__snake_case						,							encoder_attention_mask=__snake_case						,							output_hidden_states=__snake_case						,							)['hidden_states'][0]
			_UpperCamelCase :		List[str]								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							encoder_hidden_states=__snake_case						,							encoder_attention_mask=__snake_case						,							past_key_values=__snake_case						,							output_hidden_states=__snake_case						,							)['hidden_states'][0]

			# select random slice
			_UpperCamelCase :		int								=    ids_tensor((1,)						,							output_from_past.shape[-1]).item()
			_UpperCamelCase :		List[Any]								=    output_from_no_past[:, -3:, random_slice_idx].detach()
			_UpperCamelCase :		Optional[int]								=    output_from_past[:, :, random_slice_idx].detach()

			self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

			# test that outputs are equal for slice
			self.parent.assertTrue(torch.allclose(__snake_case						,							__snake_case						,							atol=1e-3))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case):
			_UpperCamelCase :		Optional[int]								=    RemBertForQuestionAnswering(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		int								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							start_positions=__snake_case						,							end_positions=__snake_case						,							)
			self.parent.assertEqual(result.start_logits.shape						,							(self.batch_size, self.seq_length))
			self.parent.assertEqual(result.end_logits.shape						,							(self.batch_size, self.seq_length))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case):
			_UpperCamelCase :		Union[str, Any]								=    self.num_labels
			_UpperCamelCase :		Dict								=    RemBertForSequenceClassification(__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		Optional[Any]								=    model(__snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							labels=__snake_case)
			self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.num_labels))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case):
			_UpperCamelCase :		str								=    self.num_labels
			_UpperCamelCase :		Tuple								=    RemBertForTokenClassification(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		int								=    model(__snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							labels=__snake_case)
			self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.seq_length, self.num_labels))


		def 		A__       (					self						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case):
			_UpperCamelCase :		Any								=    self.num_choices
			_UpperCamelCase :		Any								=    RemBertForMultipleChoice(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase :		List[str]								=    input_ids.unsqueeze(1).expand(-1						,							self.num_choices						,							-1).contiguous()
			_UpperCamelCase :		Optional[Any]								=    token_type_ids.unsqueeze(1).expand(-1						,							self.num_choices						,							-1).contiguous()
			_UpperCamelCase :		Tuple								=    input_mask.unsqueeze(1).expand(-1						,							self.num_choices						,							-1).contiguous()
			_UpperCamelCase :		List[Any]								=    model(
			    __snake_case						,							attention_mask=__snake_case						,							token_type_ids=__snake_case						,							labels=__snake_case						,							)
			self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.num_choices))






		def 		A__       (					self):
			_UpperCamelCase :		str								=    self.prepare_config_and_inputs()
			(
			    (
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		
			) :		List[Any]								=    config_and_inputs
			_UpperCamelCase :		List[Any]								=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
			return config, inputs_dict







@require_torch
class 						lowercase     (					_lowercase     ,				_lowercase     ,				unittest.TestCase				):


		"""simple docstring"""




		a__       							=	(
		    (
		        RemBertModel,
		        RemBertForMaskedLM,
		        RemBertForCausalLM,
		        RemBertForMultipleChoice,
		        RemBertForQuestionAnswering,
		        RemBertForSequenceClassification,
		        RemBertForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		a__       							=	(RemBertForCausalLM,) if is_torch_available() else ()
		a__       							=	(
		    {
		        "feature-extraction": RemBertModel,
		        "fill-mask": RemBertForMaskedLM,
		        "question-answering": RemBertForQuestionAnswering,
		        "text-classification": RemBertForSequenceClassification,
		        "text-generation": RemBertForCausalLM,
		        "token-classification": RemBertForTokenClassification,
		        "zero-shot": RemBertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)


		def 		A__       (					self):
			_UpperCamelCase :		Dict								=    RemBertModelTester(self)
			_UpperCamelCase :		List[Any]								=    ConfigTester(self						,							config_class=__snake_case						,							hidden_size=37)


		def 		A__       (					self):
			self.config_tester.run_common_tests()


		def 		A__       (					self):
			_UpperCamelCase :		Dict								=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		List[Any]								=    self.model_tester.prepare_config_and_inputs()
			for type in ["absolute", "relative_key", "relative_key_query"]:
				_UpperCamelCase :		int								=    type
				self.model_tester.create_and_check_model(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		Tuple								=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_lm(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		List[str]								=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_multiple_choice(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		int								=    self.model_tester.prepare_config_and_inputs_for_decoder()
			self.model_tester.create_and_check_decoder_model_past_large_inputs(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		Optional[int]								=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_question_answering(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		Any								=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_sequence_classification(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		Any								=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_token_classification(*__snake_case)


		def 		A__       (					self):
			_UpperCamelCase :		Union[str, Any]								=    self.model_tester.prepare_config_and_inputs_for_decoder()
			self.model_tester.create_and_check_model_as_decoder(*__snake_case)


		def 		A__       (					self):
			# This regression test was failing with PyTorch < 1.3
			(
			    (
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		(
			    _UpperCamelCase
			)						,		
			) :		Optional[int]								=    self.model_tester.prepare_config_and_inputs_for_decoder()

			_UpperCamelCase :		Optional[int]								=    None

			self.model_tester.create_and_check_model_as_decoder(
			    __snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							__snake_case						,							)






		@slow
		def 		A__       (					self):
			for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
				_UpperCamelCase :		str								=    RemBertModel.from_pretrained(__snake_case)
				self.assertIsNotNone(__snake_case)







@require_torch
class 						lowercase     (					unittest.TestCase				):


		"""simple docstring"""




		@slow
		def 		A__       (					self):
			# Test exact values at the last hidden layer
			_UpperCamelCase :		Dict								=    RemBertModel.from_pretrained('google/rembert')
			_UpperCamelCase :		Dict								=    torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]])
			_UpperCamelCase :		Union[str, Any]								=    torch.tensor([[0, 0, 0, 1, 1]])
			with torch.no_grad():
				_UpperCamelCase :		Any								=    model(__snake_case						,							token_type_ids=__snake_case						,							output_hidden_states=__snake_case)

			_UpperCamelCase :		Optional[int]								=    11_52

			_UpperCamelCase :		int								=    torch.Size((1, 5, hidden_size))
			self.assertEqual(output['last_hidden_state'].shape						,							__snake_case)

			_UpperCamelCase :		List[Any]								=    torch.tensor(
			    [
			        [
			            [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
			            [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
			            [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
			            [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
			            [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
			        ]
			    ])

			# Running on the original tf implementation gives slightly different results here.
			# Not clear why this variations is present
			# TODO: Find reason for discrepancy
			# expected_original_implementation = [[
			#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
			#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
			#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
			#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
			#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
			# ]]

			self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]						,							__snake_case						,							atol=1e-4))











"""simple docstring"""




import argparse

import torch
from clip import load

from transformers import CLIPConfig, CLIPModel





def 			a_							(				_lowercase			,		_lowercase  ):
						_UpperCamelCase  ,				_UpperCamelCase  ,				_UpperCamelCase     :   Any      =       pt_attn_layer.in_proj_weight.chunk(3			,		dim=0  )
						_UpperCamelCase  ,				_UpperCamelCase  ,				_UpperCamelCase     :   Dict      =       pt_attn_layer.in_proj_bias.chunk(3			,		dim=0  )

						_UpperCamelCase     :   int      =       pt_attn_layer.out_proj.weight
						_UpperCamelCase     :   List[Any]      =       pt_attn_layer.out_proj.bias

						_UpperCamelCase     :   Any      =       q_proj
						_UpperCamelCase     :   str      =       q_proj_bias

						_UpperCamelCase     :   str      =       k_proj
						_UpperCamelCase     :   List[str]      =       k_proj_bias

						_UpperCamelCase     :   Dict      =       v_proj
						_UpperCamelCase     :   Dict      =       v_proj_bias

						_UpperCamelCase     :   Dict      =       out_proj_weights
						_UpperCamelCase     :   Optional[Any]      =       out_proj_bias





def 			a_							(				_lowercase			,		_lowercase  ):
						copy_linear(hf_mlp.fca			,		pt_mlp.c_fc  )
						copy_linear(hf_mlp.fca			,		pt_mlp.c_proj  )





def 			a_							(				_lowercase			,		_lowercase  ):
						_UpperCamelCase     :   str      =       pt_linear.weight
						_UpperCamelCase     :   List[str]      =       pt_linear.bias





def 			a_							(				_lowercase			,		_lowercase  ):
						# copy layer norms
						copy_linear(hf_layer.layer_norma			,		pt_layer.ln_a  )
						copy_linear(hf_layer.layer_norma			,		pt_layer.ln_a  )

						# copy MLP
						copy_mlp(hf_layer.mlp			,		pt_layer.mlp  )

						# copy attn
						copy_attn_layer(hf_layer.self_attn			,		pt_layer.attn  )





def 			a_							(				_lowercase			,		_lowercase  ):
						for hf_layer, pt_layer in zip(_lowercase			,		_lowercase  ):
												copy_layer(_lowercase			,		_lowercase  )





def 			a_							(				_lowercase			,		_lowercase  ):
						# copy  embeds
						_UpperCamelCase     :   Tuple      =       pt_model.token_embedding.weight
						_UpperCamelCase     :   List[Any]      =       pt_model.positional_embedding

						# copy layer norm
						copy_linear(hf_encoder.final_layer_norm			,		pt_model.ln_final  )

						# copy hidden layers
						copy_layers(hf_encoder.encoder.layers			,		pt_model.transformer.resblocks  )





def 			a_							(				_lowercase			,		_lowercase  ):
						# copy projection
						_UpperCamelCase     :   str      =       pt_model.text_projection.data.T

						# copy text encoder
						copy_encoder(hf_model.text_model			,		_lowercase  )





def 			a_							(				_lowercase			,		_lowercase  ):
						# copy projection
						_UpperCamelCase     :   Tuple      =       pt_model.visual.proj.data.T

						# copy layer norms
						copy_linear(hf_model.vision_model.pre_layrnorm			,		pt_model.visual.ln_pre  )
						copy_linear(hf_model.vision_model.post_layernorm			,		pt_model.visual.ln_post  )

						# copy embeds
						_UpperCamelCase     :   str      =       pt_model.visual.conva.weight.data
						_UpperCamelCase     :   Dict      =       pt_model.visual.class_embedding
						_UpperCamelCase     :   Union[str, Any]      =       pt_model.visual.positional_embedding.data

						# copy encoder
						copy_layers(hf_model.vision_model.encoder.layers			,		pt_model.visual.transformer.resblocks  )





@torch.no_grad()
def 			a_							(				_lowercase			,		_lowercase			,		_lowercase=None  ):
						if config_path is not None:
												_UpperCamelCase     :   List[Any]      =       CLIPConfig.from_pretrained(_lowercase  )
						else:
												_UpperCamelCase     :   Optional[int]      =       CLIPConfig(projection_dim=512			,		text_config={}			,		vision_config={}  )

						_UpperCamelCase     :   str      =       CLIPModel(_lowercase  ).eval()

						_UpperCamelCase  ,				_UpperCamelCase     :   Optional[int]      =       load(_lowercase			,		device='''cpu'''			,		jit=_lowercase  )
						_UpperCamelCase     :   Union[str, Any]      =       pt_model.eval()

						copy_text_model_and_projection(_lowercase			,		_lowercase  )
						copy_vison_model_and_projection(_lowercase			,		_lowercase  )
						_UpperCamelCase     :   int      =       pt_model.logit_scale

						_UpperCamelCase     :   List[Any]      =       torch.arange(0			,		77  ).unsqueeze(0  )
						_UpperCamelCase     :   Optional[int]      =       torch.randn(1			,		3			,		224			,		224  )

						_UpperCamelCase     :   Tuple      =       hf_model(input_ids=_lowercase			,		pixel_values=_lowercase			,		return_dict=_lowercase  )
						_UpperCamelCase     :   Dict      =       hf_outputs.logits_per_image
						_UpperCamelCase     :   Any      =       hf_outputs.logits_per_text
						_UpperCamelCase  ,				_UpperCamelCase     :   Any      =       pt_model(_lowercase			,		_lowercase  )

						assert torch.allclose(_lowercase			,		_lowercase			,		atol=1E-3  )
						assert torch.allclose(_lowercase			,		_lowercase			,		atol=1E-3  )

						hf_model.save_pretrained(_lowercase  )


if __name__ == "__main__":
						UpperCamelCase_	 =argparse.ArgumentParser()
						parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
						parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
						parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
						UpperCamelCase_	 =parser.parse_args()

						convert_clip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)



