



import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase=None):
					UpperCamelCase_			= None
					# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
					if str(weight.dtype) == "torch.bfloat16":
										# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
										UpperCamelCase_			= weight.view(torch.intaa)
										UpperCamelCase_			= "bfloat16"
					UpperCamelCase_			= weight.cpu().numpy()
					UpperCamelCase_			= os.path.join(__snake_case ,						f"""{weight_name}.dat""")
					if index is not None:
										if dtype is None:
															UpperCamelCase_			= str(array.dtype)
										UpperCamelCase_			= {"dtype": dtype, "shape": list(array.shape)}
					if array.ndim == 0:
										UpperCamelCase_			= array[None]
					UpperCamelCase_			= np.memmap(__snake_case ,						dtype=array.dtype ,						mode="w+" ,						shape=array.shape)
					UpperCamelCase_			= array[:]
					file_array.flush()
					return index



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase):
					UpperCamelCase_			= tuple(weight_info["shape"])
					if shape == ():
										# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
										UpperCamelCase_			= (1,)

					UpperCamelCase_			= weight_info["dtype"]
					if dtype == "bfloat16":
										# NumPy does not support bfloat16 so this was saved as a int16
										UpperCamelCase_			= "int16"

					UpperCamelCase_			= np.memmap(__snake_case ,						dtype=__snake_case ,						shape=__snake_case ,						mode="r")

					if len(weight_info["shape"]) == 0:
										UpperCamelCase_			= weight[0]
					UpperCamelCase_			= torch.tensor(__snake_case)
					if weight_info["dtype"] == "bfloat16":
										UpperCamelCase_			= weight.view(torch.bfloataa)

					return weight



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase):


					if index is None or len(__snake_case) == 0:
										# Nothing to save
										return

					UpperCamelCase_			= os.path.join(__snake_case ,						"index.json")
					if os.path.isfile(__snake_case):
										with open(__snake_case ,						"r" ,						encoding="utf-8") as f:
															UpperCamelCase_			= json.load(__snake_case)
					else:
										UpperCamelCase_			= {}
					current_index.update(__snake_case)

					with open(__snake_case ,						"w" ,						encoding="utf-8") as f:
										json.dump(__snake_case ,						__snake_case ,						indent=2)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase):
					os.makedirs(__snake_case ,						exist_ok=__snake_case)
					UpperCamelCase_			= {}
					for name, parameter in state_dict.items():
										UpperCamelCase_			= offload_weight(__snake_case ,						__snake_case ,						__snake_case ,						index=__snake_case)

					# Update index
					save_offload_index(__snake_case ,						__snake_case)


class    _lowercase (lowerCamelCase_   ):
				'''simple docstring'''







				def __init__(			self  ,			snake_case__  ,			snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_			= dataset
									UpperCamelCase_			= prefix

				def __getitem__(			self  ,			snake_case__     ):

									'''simple docstring'''


									return self.dataset[F"""{self.prefix}{key}"""]

				def __iter__(			self     ):

									'''simple docstring'''


									return iter([key for key in self.dataset if key.startswith(self.prefix     )]     )

				def __len__(			self     ):

									'''simple docstring'''


									return len(self.dataset     )




class    _lowercase (lowerCamelCase_   ):
				'''simple docstring'''







				def __init__(			self  ,			snake_case__ = None  ,			snake_case__ = None  ,			snake_case__ = None  ,			snake_case__=None  ,			):

									'''simple docstring'''


									if state_dict is None and save_folder is None:
														raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights."     )

									UpperCamelCase_			= {} if state_dict is None else state_dict
									UpperCamelCase_			= save_folder
									if index is None and save_folder is not None:
														with open(os.path.join(__UpperCAmelCase  ,			"index.json"     )     ) as f:
																			UpperCamelCase_			= json.load(__UpperCAmelCase     )
									UpperCamelCase_			= {} if index is None else index
									UpperCamelCase_			= list(self.state_dict.keys()     )
									self.all_keys.extend([key for key in self.index if key not in self.all_keys]     )
									UpperCamelCase_			= device

				def __getitem__(			self  ,			snake_case__     ):

									'''simple docstring'''


									if key in self.state_dict:
														return self.state_dict[key]
									UpperCamelCase_			= self.index[key]
									if weight_info.get("safetensors_file"     ) is not None:
														if not is_safetensors_available():
																			raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`."     )

														from safetensors import safe_open

														UpperCamelCase_			= "cpu" if self.device is None else self.device
														with safe_open(weight_info["safetensors_file"]  ,			framework="pt"  ,			device=__UpperCAmelCase     ) as f:
																			UpperCamelCase_			= f.get_tensor(weight_info.get("weight_name"  ,			__UpperCAmelCase     )     )

														if "dtype" in weight_info:
																			return tensor.to(getattr(__UpperCAmelCase  ,			weight_info["dtype"]     )     )
														else:
																			return tensor

									UpperCamelCase_			= os.path.join(self.save_folder  ,			F"""{key}.dat"""     )
									return load_offloaded_weight(__UpperCAmelCase  ,			__UpperCAmelCase     )

				def __iter__(			self     ):

									'''simple docstring'''


									return iter(self.all_keys     )

				def __len__(			self     ):

									'''simple docstring'''


									return len(self.all_keys     )




def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase):
					UpperCamelCase_			= {}
					for module_name in submodule_names:
										# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
										# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
										result.update(
										    {
										        key: param
										        for key, param in state_dict.items()
										        if key == module_name or key.startswith(module_name + ".")
										    })
					return result






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import CLIPImageProcessor




class     A					(				unittest.TestCase       ):







							def __init__(    self   :	Any   ,     __UpperCAmelCase   :	Optional[int]   ,     __UpperCAmelCase   :	List[Any]=7   ,     __UpperCAmelCase   :	Union[str, Any]=3   ,     __UpperCAmelCase   :	Optional[Any]=18   ,     __UpperCAmelCase   :	List[str]=30   ,     __UpperCAmelCase   :	int=400   ,     __UpperCAmelCase   :	List[Any]=True   ,     __UpperCAmelCase   :	Tuple=None   ,     __UpperCAmelCase   :	Optional[Any]=True   ,     __UpperCAmelCase   :	Any=None   ,     __UpperCAmelCase   :	Optional[int]=True   ,     __UpperCAmelCase   :	List[str]=[0.48_145_466, 0.4_578_275, 0.40_821_073]   ,     __UpperCAmelCase   :	List[Any]=[0.26_862_954, 0.26_130_258, 0.27_577_711]   ,     __UpperCAmelCase   :	Dict=True   ,     )		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= size if size is not None else {'shortest_edge': 20}
										UpperCamelCase_     				= crop_size if crop_size is not None else {'height': 18, 'width': 18}
										UpperCamelCase_     				= parent
										UpperCamelCase_     				= batch_size
										UpperCamelCase_     				= num_channels
										UpperCamelCase_     				= image_size
										UpperCamelCase_     				= min_resolution
										UpperCamelCase_     				= max_resolution
										UpperCamelCase_     				= do_resize
										UpperCamelCase_     				= size
										UpperCamelCase_     				= do_center_crop
										UpperCamelCase_     				= crop_size
										UpperCamelCase_     				= do_normalize
										UpperCamelCase_     				= image_mean
										UpperCamelCase_     				= image_std
										UpperCamelCase_     				= do_convert_rgb







							def        lowercase__							(    self   :	Tuple				)		->					List[Any]:

										"""simple docstring"""



										return {
										    "do_resize": self.do_resize,
										    "size": self.size,
										    "do_center_crop": self.do_center_crop,
										    "crop_size": self.crop_size,
										    "do_normalize": self.do_normalize,
										    "image_mean": self.image_mean,
										    "image_std": self.image_std,
										    "do_convert_rgb": self.do_convert_rgb,
										}







							def        lowercase__							(    self   :	List[Any]   ,     __UpperCAmelCase   :	str=False   ,     __UpperCAmelCase   :	Tuple=False   ,     __UpperCAmelCase   :	Optional[int]=False				)		->					Any:

										"""simple docstring"""



										assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

										if equal_resolution:
													UpperCamelCase_     				= []
													for i in range(self.batch_size				):
																image_inputs.append(
																    np.random.randint(
																        255   ,     size=(self.num_channels, self.max_resolution, self.max_resolution)   ,     dtype=np.uinta				)				)
										else:
													UpperCamelCase_     				= []
													for i in range(self.batch_size				):
																UpperCamelCase_				,   UpperCamelCase_     				= np.random.choice(np.arange(self.min_resolution   ,     self.max_resolution				)   ,     2				)
																image_inputs.append(np.random.randint(255   ,     size=(self.num_channels, width, height)   ,     dtype=np.uinta				)				)

										if not numpify and not torchify:
													# PIL expects the channel dimension as last dimension
													UpperCamelCase_     				= [Image.fromarray(np.moveaxis(__UpperCAmelCase   ,     0   ,     -1				)				) for x in image_inputs]

										if torchify:
													UpperCamelCase_     				= [torch.from_numpy(__UpperCAmelCase				) for x in image_inputs]

										return image_inputs







@require_torch
@require_vision
class     A					(				lowerCamelCase_  ,						unittest.TestCase       ):
							_SCREAMING_SNAKE_CASE  :		Any			  =       CLIPImageProcessor if is_vision_available() else None







							def        lowercase__							(    self   :	Dict				)		->					List[str]:

										"""simple docstring"""



										UpperCamelCase_     				= CLIPImageProcessingTester(self				)







							@property
							def        lowercase__							(    self   :	Optional[int]				)		->					Optional[Any]:

										"""simple docstring"""



										return self.image_processor_tester.prepare_image_processor_dict()







							def        lowercase__							(    self   :	Tuple				)		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_resize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'size'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_normalize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_mean'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_std'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_convert_rgb'				)				)







							def        lowercase__							(    self   :	Optional[int]				)		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class.from_dict(self.image_processor_dict				)
										self.assertEqual(image_processor.size   ,     {'shortest_edge': 20}				)
										self.assertEqual(image_processor.crop_size   ,     {'height': 18, 'width': 18}				)

										UpperCamelCase_     				= self.image_processing_class.from_dict(self.image_processor_dict   ,     size=42   ,     crop_size=84				)
										self.assertEqual(image_processor.size   ,     {'shortest_edge': 42}				)
										self.assertEqual(image_processor.crop_size   ,     {'height': 84, 'width': 84}				)







							def        lowercase__							(    self   :	Union[str, Any]				)		->					Optional[int]:

										"""simple docstring"""



										pass







							def        lowercase__							(    self   :	int				)		->					Optional[int]:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random PIL images
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     Image.Image				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )







							def        lowercase__							(    self   :	Optional[Any]				)		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random numpy tensors
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase   ,     numpify=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     np.ndarray				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )







							def        lowercase__							(    self   :	Optional[Any]				)		->					Any:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random PyTorch tensors
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase   ,     torchify=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     torch.Tensor				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )







@require_torch
@require_vision
class     A					(				lowerCamelCase_  ,						unittest.TestCase       ):
							_SCREAMING_SNAKE_CASE  :		str			  =       CLIPImageProcessor if is_vision_available() else None







							def        lowercase__							(    self   :	str				)		->					Any:

										"""simple docstring"""



										UpperCamelCase_     				= CLIPImageProcessingTester(self   ,     num_channels=4				)
										UpperCamelCase_     				= 3







							@property
							def        lowercase__							(    self   :	Dict				)		->					str:

										"""simple docstring"""



										return self.image_processor_tester.prepare_image_processor_dict()







							def        lowercase__							(    self   :	Optional[int]				)		->					List[str]:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_resize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'size'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_normalize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_mean'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_std'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_convert_rgb'				)				)







							def        lowercase__							(    self   :	int				)		->					Any:

										"""simple docstring"""



										pass







							def        lowercase__							(    self   :	Any				)		->					Optional[int]:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random PIL images
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     Image.Image				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.expected_encoded_image_num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.expected_encoded_image_num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )
