


'''simple docstring'''


import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)







def 			snake_case_   (  SCREAMING_SNAKE_CASE__     ):







				"""simple docstring"""





				if model.config.model_type == "gpt2":
								return model.transformer.h[0].mlp.c_fc
				return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
				import torch
				import torch.nn as nn





				class     lowercase__  (						nn.Module					):





											'''simple docstring'''


											def __init__(    self    ,					__snake_case    ,					__snake_case  ):
															super().__init__()
															_SCREAMING_SNAKE_CASE :       Tuple     		=      module
															_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      nn.Sequential(
															    nn.Linear(module.in_features    ,					__snake_case    ,					bias=__snake_case  )    ,					nn.Linear(__snake_case    ,					module.out_features    ,					bias=__snake_case  )    ,					)
															_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      (2.0 / (5 * min(module.in_features    ,					module.out_features  ))) ** 0.5
															nn.init.normal_(self.adapter[0].weight    ,					std=__snake_case  )
															nn.init.zeros_(self.adapter[1].weight  )
															self.adapter.to(module.weight.device  )






											def UpperCAmelCase_	(    self    ,					__snake_case    ,					*__snake_case    ,					**__snake_case  ):
															return self.module(__snake_case    ,					*__snake_case    ,					**__snake_case  ) + self.adapter(__snake_case  )





@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class     lowercase__  (						unittest.TestCase					):





							'''simple docstring'''


							A_							:      Any      	=  """bigscience/bloom-1b7"""

							# Constant values
							A_							:      Optional[int]      	=  2.109659552692574

							A_							:      List[str]      	=  """Hello my name is"""
							A_							:      Any      	=  set()
							EXPECTED_OUTPUTS.add("""Hello my name is John and I am a professional photographer. I"""					)
							EXPECTED_OUTPUTS.add("""Hello my name is John.\nI am a friend of your father.\n"""					)
							EXPECTED_OUTPUTS.add("""Hello my name is John Doe, I am a student at the University"""					)
							A_							:      List[str]      	=  10






							def UpperCAmelCase_	(    self  ):
											# Models and tokenizer
											_SCREAMING_SNAKE_CASE :       Any     		=      AutoTokenizer.from_pretrained(self.model_name  )





class     lowercase__  (						_snake_case					):





							'''simple docstring'''


							def UpperCAmelCase_	(    self  ):
											super().setUp()

											# Models and tokenizer
											_SCREAMING_SNAKE_CASE :       Tuple     		=      AutoModelForCausalLM.from_pretrained(
											    self.model_name    ,					torch_dtype=torch.floataa    ,					device_map="""auto"""  )
											_SCREAMING_SNAKE_CASE :       str     		=      AutoModelForCausalLM.from_pretrained(self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )






							def UpperCAmelCase_	(    self  ):
											del self.model_fpaa
											del self.model_abit

											gc.collect()
											torch.cuda.empty_cache()






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       str     		=      self.model_abit.config

											self.assertTrue(hasattr(__snake_case    ,					"""quantization_config"""  )  )

											_SCREAMING_SNAKE_CASE :       List[str]     		=      config.to_dict()
											_SCREAMING_SNAKE_CASE :       List[str]     		=      config.to_diff_dict()

											_SCREAMING_SNAKE_CASE :       Dict     		=      config.to_json_string()






							def UpperCAmelCase_	(    self  ):
											from bitsandbytes.nn import Paramsabit

											_SCREAMING_SNAKE_CASE :       List[Any]     		=      self.model_fpaa.get_memory_footprint()
											_SCREAMING_SNAKE_CASE :       Dict     		=      self.model_abit.get_memory_footprint()

											self.assertAlmostEqual(mem_fpaa / mem_abit    ,					self.EXPECTED_RELATIVE_DIFFERENCE  )
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      get_some_linear_layer(self.model_abit  )
											self.assertTrue(linear.weight.__class__ == Paramsabit  )






							def UpperCAmelCase_	(    self  ):
											from transformers import TaPreTrainedModel

											self.model_fpaa.get_memory_footprint()
											self.model_abit.get_memory_footprint()

											for name, module in self.model_abit.named_modules():
															if isinstance(__snake_case    ,					torch.nn.Linear  ):
																			if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
																							# 4-bit parameters are packed in uint8 variables
																							self.assertTrue(module.weight.dtype == torch.uinta  )






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  )
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      self.model_abit.generate(input_ids=encoded_input["""input_ids"""].to(0  )    ,					max_new_tokens=10  )

											self.assertIn(self.tokenizer.decode(output_sequences[0]    ,					skip_special_tokens=__snake_case  )    ,					self.EXPECTED_OUTPUTS  )






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      BitsAndBytesConfig()
											_SCREAMING_SNAKE_CASE :       Tuple     		=      True

											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      AutoModelForCausalLM.from_pretrained(
											    self.model_name    ,					quantization_config=__snake_case    ,					device_map="""auto"""  )

											_SCREAMING_SNAKE_CASE :       List[str]     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  )
											_SCREAMING_SNAKE_CASE :       Dict     		=      model_abit_from_config.generate(
											    input_ids=encoded_input["""input_ids"""].to(0  )    ,					max_new_tokens=10  )

											self.assertIn(self.tokenizer.decode(output_sequences[0]    ,					skip_special_tokens=__snake_case  )    ,					self.EXPECTED_OUTPUTS  )






							def UpperCAmelCase_	(    self  ):
											with self.assertRaises(__snake_case  ), tempfile.TemporaryDirectory() as tmpdirname:
															self.model_abit.save_pretrained(__snake_case  )






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      BitsAndBytesConfig()

											with self.assertRaises(__snake_case  ):
															_SCREAMING_SNAKE_CASE :       Optional[int]     		=      AutoModelForCausalLM.from_pretrained(
															    self.model_name    ,					quantization_config=__snake_case    ,					load_in_abit=__snake_case    ,					device_map="""auto"""    ,					bnb_abit_quant_type="""nf4"""    ,					)






							def UpperCAmelCase_	(    self  ):
											with self.assertRaises(__snake_case  ):
															# Tries with `str`
															self.model_abit.to("""cpu"""  )

											with self.assertRaises(__snake_case  ):
															# Tries with a `dtype``
															self.model_abit.to(torch.floataa  )

											with self.assertRaises(__snake_case  ):
															# Tries with a `device`
															self.model_abit.to(torch.device("""cuda:0"""  )  )

											with self.assertRaises(__snake_case  ):
															# Tries with a `device`
															self.model_abit.float()

											with self.assertRaises(__snake_case  ):
															# Tries with a `device`
															self.model_abit.half()

											# Test if we did not break anything
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  )

											_SCREAMING_SNAKE_CASE :       int     		=      self.model_fpaa.to(torch.floataa  )
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      self.model_fpaa.generate(input_ids=encoded_input["""input_ids"""].to(0  )    ,					max_new_tokens=10  )

											# Check this does not throw an error
											_SCREAMING_SNAKE_CASE :       str     		=      self.model_fpaa.to("""cpu"""  )

											# Check this does not throw an error
											_SCREAMING_SNAKE_CASE :       List[str]     		=      self.model_fpaa.half()

											# Check this does not throw an error
											_SCREAMING_SNAKE_CASE :       Any     		=      self.model_fpaa.float()






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       List[str]     		=      AutoModelForSeqaSeqLM.from_pretrained("""t5-small"""    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa  )





@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class     lowercase__  (						unittest.TestCase					):





							'''simple docstring'''


							@classmethod
							def UpperCAmelCase_	(    cls  ):
											_SCREAMING_SNAKE_CASE :       Any     		=      """t5-small"""
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      """google/flan-t5-small"""  # flan-t5 uses dense-act instead of dense-relu-dense
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      AutoTokenizer.from_pretrained(cls.model_name  )
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      """Translate in German: Hello, my dog is cute"""






							def UpperCAmelCase_	(    self  ):
											gc.collect()
											torch.cuda.empty_cache()






							def UpperCAmelCase_	(    self  ):
											from transformers import TaForConditionalGeneration

											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      TaForConditionalGeneration._keep_in_fpaa_modules
											_SCREAMING_SNAKE_CASE :       List[str]     		=      None

											# test with `t5-small`
											_SCREAMING_SNAKE_CASE :       Any     		=      TaForConditionalGeneration.from_pretrained(self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  ).to(0  )
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      model.generate(**__snake_case  )

											# test with `flan-t5-small`
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      TaForConditionalGeneration.from_pretrained(
											    self.dense_act_model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											_SCREAMING_SNAKE_CASE :       Any     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  ).to(0  )
											_SCREAMING_SNAKE_CASE :       List[str]     		=      model.generate(**__snake_case  )
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      modules






							def UpperCAmelCase_	(    self  ):
											import bitsandbytes as bnb

											from transformers import TaForConditionalGeneration

											# test with `t5-small`
											_SCREAMING_SNAKE_CASE :       Dict     		=      TaForConditionalGeneration.from_pretrained(self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )

											# there was a bug with decoders - this test checks that it is fixed
											self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q    ,					bnb.nn.Linearabit  )  )

											_SCREAMING_SNAKE_CASE :       str     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  ).to(0  )
											_SCREAMING_SNAKE_CASE :       Tuple     		=      model.generate(**__snake_case  )

											# test with `flan-t5-small`
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      TaForConditionalGeneration.from_pretrained(
											    self.dense_act_model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  ).to(0  )
											_SCREAMING_SNAKE_CASE :       int     		=      model.generate(**__snake_case  )





class     lowercase__  (						_snake_case					):





							'''simple docstring'''


							def UpperCAmelCase_	(    self  ):
											super().setUp()
											# model_name
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      """bigscience/bloom-560m"""
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      """t5-small"""

											# Different types of model

											_SCREAMING_SNAKE_CASE :       Dict     		=      AutoModel.from_pretrained(self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											# Sequence classification model
											_SCREAMING_SNAKE_CASE :       Any     		=      AutoModelForSequenceClassification.from_pretrained(
											    self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											# CausalLM model
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      AutoModelForCausalLM.from_pretrained(self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )
											# Seq2seq model
											_SCREAMING_SNAKE_CASE :       List[str]     		=      AutoModelForSeqaSeqLM.from_pretrained(
											    self.seq_to_seq_name    ,					load_in_abit=__snake_case    ,					device_map="""auto"""  )






							def UpperCAmelCase_	(    self  ):
											del self.base_model
											del self.sequence_model
											del self.model_abit
											del self.seq_to_seq_model

											gc.collect()
											torch.cuda.empty_cache()






							def UpperCAmelCase_	(    self  ):
											from bitsandbytes.nn import Paramsabit

											self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit  )

											# Other heads should be nn.Parameter
											self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter  )
											self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter  )
											self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter  )





class     lowercase__  (						_snake_case					):





							'''simple docstring'''


							def UpperCAmelCase_	(    self  ):
											super().setUp()






							def UpperCAmelCase_	(    self  ):
											del self.pipe

											gc.collect()
											torch.cuda.empty_cache()






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      pipeline(
											    """text-generation"""    ,					model=self.model_name    ,					model_kwargs={"""device_map""": """auto""", """load_in_4bit""": True, """torch_dtype""": torch.floataa}    ,					max_new_tokens=self.MAX_NEW_TOKENS    ,					)

											# Real second forward pass
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      self.pipe(self.input_text  )
											self.assertIn(pipeline_output[0]["""generated_text"""]    ,					self.EXPECTED_OUTPUTS  )





@require_torch_multi_gpu
class     lowercase__  (						_snake_case					):





							'''simple docstring'''


							def UpperCAmelCase_	(    self  ):
											super().setUp()






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       str     		=      AutoModelForCausalLM.from_pretrained(
											    self.model_name    ,					load_in_abit=__snake_case    ,					device_map="""balanced"""  )

											# Check correct device map
											self.assertEqual(set(model_parallel.hf_device_map.values()  )    ,					{0, 1}  )

											# Check that inference pass works on the model
											_SCREAMING_SNAKE_CASE :       int     		=      self.tokenizer(self.input_text    ,					return_tensors="""pt"""  )

											# Second real batch
											_SCREAMING_SNAKE_CASE :       List[str]     		=      model_parallel.generate(input_ids=encoded_input["""input_ids"""].to(0  )    ,					max_new_tokens=10  )
											self.assertIn(self.tokenizer.decode(output_parallel[0]    ,					skip_special_tokens=__snake_case  )    ,					self.EXPECTED_OUTPUTS  )





class     lowercase__  (						_snake_case					):





							'''simple docstring'''


							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       int     		=      """facebook/opt-350m"""
											super().setUp()






							def UpperCAmelCase_	(    self  ):
											if version.parse(importlib.metadata.version("""bitsandbytes"""  )  ) < version.parse("""0.37.0"""  ):
															return

											# Step 1: freeze all parameters
											_SCREAMING_SNAKE_CASE :       List[str]     		=      AutoModelForCausalLM.from_pretrained(self.model_name    ,					load_in_abit=__snake_case  )

											self.assertEqual(set(model.hf_device_map.values()  )    ,					{torch.cuda.current_device()}  )

											for param in model.parameters():
															_SCREAMING_SNAKE_CASE :       int     		=      False  # freeze the model - train adapters later
															if param.ndim == 1:
																			# cast the small parameters (e.g. layernorm) to fp32 for stability
																			_SCREAMING_SNAKE_CASE :       List[Any]     		=      param.data.to(torch.floataa  )

        # Step 2: add adapters
											for _, module in model.named_modules():
															if "OPTAttention" in repr(type(__snake_case  )  ):
																			_SCREAMING_SNAKE_CASE :       Any     		=      LoRALayer(module.q_proj    ,					rank=16  )
																			_SCREAMING_SNAKE_CASE :       Optional[int]     		=      LoRALayer(module.k_proj    ,					rank=16  )
																			_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      LoRALayer(module.v_proj    ,					rank=16  )

        # Step 3: dummy batch
											_SCREAMING_SNAKE_CASE :       int     		=      self.tokenizer("""Test batch """    ,					return_tensors="""pt"""  ).to(0  )

											# Step 4: Check if the gradient is not None
											with torch.cuda.amp.autocast():
															_SCREAMING_SNAKE_CASE :       Tuple     		=      model.forward(**__snake_case  )
															out.logits.norm().backward()

											for module in model.modules():
															if isinstance(__snake_case    ,					__snake_case  ):
																			self.assertTrue(module.adapter[1].weight.grad is not None  )
																			self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0  )
															elif isinstance(__snake_case    ,					nn.Embedding  ):
																			self.assertTrue(module.weight.grad is None  )





class     lowercase__  (						_snake_case					):





							'''simple docstring'''


							A_							:      Any      	=  """gpt2-xl"""
							A_							:      Optional[int]      	=  3.3191854854152187








'''simple docstring'''


import copy
import inspect
import unittest

from transformers import AutoBackbone
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import require_timm, require_torch, torch_device
from transformers.utils.import_utils import is_torch_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor


if is_torch_available():
				import torch

				from transformers import TimmBackbone, TimmBackboneConfig

from ...test_pipeline_mixin import PipelineTesterMixin





class     lowercase__  :





							'''simple docstring'''


							def __init__(    self    ,					__snake_case    ,					__snake_case=None    ,					__snake_case=None    ,					__snake_case=None    ,					__snake_case="resnet50"    ,					__snake_case=3    ,					__snake_case=32    ,					__snake_case=3    ,					__snake_case=True    ,					__snake_case=True    ,					):
											_SCREAMING_SNAKE_CASE :       Tuple     		=      parent
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      out_indices if out_indices is not None else [4]
											_SCREAMING_SNAKE_CASE :       str     		=      stage_names
											_SCREAMING_SNAKE_CASE :       List[str]     		=      out_features
											_SCREAMING_SNAKE_CASE :       int     		=      backbone
											_SCREAMING_SNAKE_CASE :       Any     		=      batch_size
											_SCREAMING_SNAKE_CASE :       List[str]     		=      image_size
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      num_channels
											_SCREAMING_SNAKE_CASE :       int     		=      use_pretrained_backbone
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      is_training






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       int     		=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]  )
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      self.get_config()

											return config, pixel_values






							def UpperCAmelCase_	(    self  ):
											return TimmBackboneConfig(
											    image_size=self.image_size    ,					num_channels=self.num_channels    ,					out_features=self.out_features    ,					out_indices=self.out_indices    ,					stage_names=self.stage_names    ,					use_pretrained_backbone=self.use_pretrained_backbone    ,					backbone=self.backbone    ,					)






							def UpperCAmelCase_	(    self    ,					__snake_case    ,					__snake_case  ):
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      TimmBackbone(config=__snake_case  )
											model.to(__snake_case  )
											model.eval()
											with torch.no_grad():
															_SCREAMING_SNAKE_CASE :       List[Any]     		=      model(__snake_case  )
											self.parent.assertEqual(
											    result.feature_map[-1].shape    ,					(self.batch_size, model.channels[-1], 14, 14)    ,					)






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       Tuple     		=      self.prepare_config_and_inputs()
											_SCREAMING_SNAKE_CASE		,       _SCREAMING_SNAKE_CASE :       Any     		=      config_and_inputs
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      {"""pixel_values""": pixel_values}
											return config, inputs_dict





@require_torch
@require_timm
class     lowercase__  (						_snake_case     ,     _snake_case     ,     _snake_case     ,     unittest.TestCase					):





							'''simple docstring'''


							A_							:      Optional[Any]      	=  (TimmBackbone,) if is_torch_available() else ()
							A_							:      Tuple      	=  {"""feature-extraction""": TimmBackbone} if is_torch_available() else {}
							A_							:      Optional[Any]      	=  False
							A_							:      List[Any]      	=  False
							A_							:      Dict      	=  False
							A_							:      int      	=  False






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       Any     		=      TimmBackboneModelTester(self  )
											_SCREAMING_SNAKE_CASE :       int     		=      ConfigTester(self    ,					config_class=__snake_case    ,					has_text_modality=__snake_case  )






							def UpperCAmelCase_	(    self  ):
											self.config_tester.create_and_test_config_to_json_string()
											self.config_tester.create_and_test_config_to_json_file()
											self.config_tester.create_and_test_config_from_and_save_pretrained()
											self.config_tester.create_and_test_config_with_num_labels()
											self.config_tester.check_config_can_be_init_without_params()
											self.config_tester.check_config_arguments_init()






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      """resnet18"""
											_SCREAMING_SNAKE_CASE :       Tuple     		=      """microsoft/resnet-18"""

											_SCREAMING_SNAKE_CASE :       List[str]     		=      AutoBackbone.from_pretrained(__snake_case    ,					use_timm_backbone=__snake_case  )
											_SCREAMING_SNAKE_CASE :       Tuple     		=      AutoBackbone.from_pretrained(__snake_case  )

											self.assertEqual(len(timm_model.out_features  )    ,					len(transformers_model.out_features  )  )
											self.assertEqual(len(timm_model.stage_names  )    ,					len(transformers_model.stage_names  )  )
											self.assertEqual(timm_model.channels    ,					transformers_model.channels  )
											# Out indices are set to the last layer by default. For timm models, we don't know
											# the number of layers in advance, so we set it to (-1,), whereas for transformers
											# models, we set it to [len(stage_names) - 1] (kept for backward compatibility).
											self.assertEqual(timm_model.out_indices    ,					(-1,)  )
											self.assertEqual(transformers_model.out_indices    ,					[len(timm_model.stage_names  ) - 1]  )

											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      AutoBackbone.from_pretrained(__snake_case    ,					use_timm_backbone=__snake_case    ,					out_indices=[1, 2, 3]  )
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      AutoBackbone.from_pretrained(__snake_case    ,					out_indices=[1, 2, 3]  )

											self.assertEqual(timm_model.out_indices    ,					transformers_model.out_indices  )
											self.assertEqual(len(timm_model.out_features  )    ,					len(transformers_model.out_features  )  )
											self.assertEqual(timm_model.channels    ,					transformers_model.channels  )






							@unittest.skip("""TimmBackbone doesn't support feed forward chunking"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone doesn't have num_hidden_layers attribute"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone initialization is managed on the timm side"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone models doesn't have inputs_embeds"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone models doesn't have inputs_embeds"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone model cannot be created without specifying a backbone checkpoint"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""Only checkpoints on timm can be loaded into TimmBackbone"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""model weights aren't tied in TimmBackbone."""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""model weights aren't tied in TimmBackbone."""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""Only checkpoints on timm can be loaded into TimmBackbone"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""Only checkpoints on timm can be loaded into TimmBackbone"""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone doesn't have hidden size info in its configuration."""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""TimmBackbone doesn't support output_attentions."""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""Safetensors is not supported by timm."""  )
							def UpperCAmelCase_	(    self  ):
											pass






							@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""  )
							def UpperCAmelCase_	(    self  ):
											pass






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE		,       _SCREAMING_SNAKE_CASE :       List[Any]     		=      self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															_SCREAMING_SNAKE_CASE :       List[str]     		=      model_class(__snake_case  )
															_SCREAMING_SNAKE_CASE :       Tuple     		=      inspect.signature(model.forward  )
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															_SCREAMING_SNAKE_CASE :       int     		=      [*signature.parameters.keys()]

															_SCREAMING_SNAKE_CASE :       List[Any]     		=      ["""pixel_values"""]
															self.assertListEqual(arg_names[:1]    ,					__snake_case  )






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE		,       _SCREAMING_SNAKE_CASE :       List[Any]     		=      self.model_tester.prepare_config_and_inputs_for_common()
											_SCREAMING_SNAKE_CASE :       Tuple     		=      True
											_SCREAMING_SNAKE_CASE :       List[str]     		=      self.has_attentions

											# no need to test all models as different heads yield the same functionality
											_SCREAMING_SNAKE_CASE :       str     		=      self.all_model_classes[0]
											_SCREAMING_SNAKE_CASE :       str     		=      model_class(__snake_case  )
											model.to(__snake_case  )

											_SCREAMING_SNAKE_CASE :       Tuple     		=      self._prepare_for_class(__snake_case    ,					__snake_case  )
											_SCREAMING_SNAKE_CASE :       Tuple     		=      model(**__snake_case  )
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      outputs[0][-1]

											# Encoder-/Decoder-only models
											_SCREAMING_SNAKE_CASE :       str     		=      outputs.hidden_states[0]
											hidden_states.retain_grad()

											if self.has_attentions:
															_SCREAMING_SNAKE_CASE :       Optional[int]     		=      outputs.attentions[0]
															attentions.retain_grad()

											output.flatten()[0].backward(retain_graph=__snake_case  )

											self.assertIsNotNone(hidden_states.grad  )

											if self.has_attentions:
															self.assertIsNotNone(attentions.grad  )






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE		,       _SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															_SCREAMING_SNAKE_CASE :       str     		=      model_class(__snake_case  )
															model.to(__snake_case  )
															model.eval()
															_SCREAMING_SNAKE_CASE :       List[str]     		=      model(**__snake_case  )

															self.assertEqual(len(result.feature_maps  )    ,					len(config.out_indices  )  )
															self.assertEqual(len(model.channels  )    ,					len(config.out_indices  )  )

															# Check output of last stage is taken if out_features=None, out_indices=None
															_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      copy.deepcopy(__snake_case  )
															_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      None
															_SCREAMING_SNAKE_CASE :       Tuple     		=      model_class(__snake_case  )
															model.to(__snake_case  )
															model.eval()
															_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      model(**__snake_case  )

															self.assertEqual(len(result.feature_maps  )    ,					1  )
															self.assertEqual(len(model.channels  )    ,					1  )

															# Check backbone can be initialized with fresh weights
															_SCREAMING_SNAKE_CASE :       str     		=      copy.deepcopy(__snake_case  )
															_SCREAMING_SNAKE_CASE :       Tuple     		=      False
															_SCREAMING_SNAKE_CASE :       Optional[int]     		=      model_class(__snake_case  )
															model.to(__snake_case  )
															model.eval()
															_SCREAMING_SNAKE_CASE :       List[Any]     		=      model(**__snake_case  )




