






"""simple docstring"""
import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available


def        A	(							__snake_case:   List[Any]   ,    __snake_case:   List[Any]   ,    __snake_case:   List[Any]   ,    __snake_case:   Optional[int]=None     )						->		List[str]:

       """simple docstring"""





       __magic_name__										=						None
       # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
       if str(weight.dtype     ) == "torch.bfloat16":
              # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
              __magic_name__										=						weight.view(torch.intaa     )
              __magic_name__										=						'bfloat16'
       __magic_name__										=						weight.cpu().numpy()
       __magic_name__										=						os.path.join(__snake_case   ,    F"""{weight_name}.dat"""     )
       if index is not None:
              if dtype is None:
                     __magic_name__										=						str(array.dtype     )
              __magic_name__										=						{'dtype': dtype, 'shape': list(array.shape     )}
       if array.ndim == 0:
              __magic_name__										=						array[None]
       __magic_name__										=						np.memmap(__snake_case   ,    dtype=array.dtype   ,    mode='w+'   ,    shape=array.shape     )
       __magic_name__										=						array[:]
       file_array.flush()
       return index


def        A	(							__snake_case:   int   ,    __snake_case:   str     )						->		Union[str, Any]:

       """simple docstring"""





       __magic_name__										=						tuple(weight_info['shape']     )
       if shape == ():
              # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
              __magic_name__										=						(1,)

       __magic_name__										=						weight_info['dtype']
       if dtype == "bfloat16":
              # NumPy does not support bfloat16 so this was saved as a int16
              __magic_name__										=						'int16'

       __magic_name__										=						np.memmap(__snake_case   ,    dtype=__snake_case   ,    shape=__snake_case   ,    mode='r'     )

       if len(weight_info['shape']     ) == 0:
              __magic_name__										=						weight[0]
       __magic_name__										=						torch.tensor(__snake_case     )
       if weight_info["dtype"] == "bfloat16":
              __magic_name__										=						weight.view(torch.bfloataa     )

       return weight


def        A	(							__snake_case:   int   ,    __snake_case:   str     )						->		int:

       """simple docstring"""





       if index is None or len(__snake_case     ) == 0:
              # Nothing to save
              return

       __magic_name__										=						os.path.join(__snake_case   ,    'index.json'     )
       if os.path.isfile(__snake_case     ):
              with open(__snake_case   ,    'r'   ,    encoding='utf-8'     ) as f:
                     __magic_name__										=						json.load(__snake_case     )
       else:
              __magic_name__										=						{}
       current_index.update(__snake_case     )

       with open(__snake_case   ,    'w'   ,    encoding='utf-8'     ) as f:
              json.dump(__snake_case   ,    __snake_case   ,    indent=2     )


def        A	(							__snake_case:   Union[str, os.PathLike]   ,    __snake_case:   Dict[str, torch.Tensor]     )						->		int:

       """simple docstring"""





       os.makedirs(__snake_case   ,    exist_ok=__snake_case     )
       __magic_name__										=						{}
       for name, parameter in state_dict.items():
              __magic_name__										=						offload_weight(__snake_case   ,    __snake_case   ,    __snake_case   ,    index=__snake_case     )

       # Update index
       save_offload_index(__snake_case   ,    __snake_case     )



class 					UpperCamelCase__   (							a_):







    """simple docstring"""







    def __init__(		self   :  Optional[Any]							,						UpperCamelCase_   :  Mapping							,						UpperCamelCase_   :  str   ):





           '''simple docstring'''

           __magic_name__										=						dataset
           __magic_name__										=						prefix







    def __getitem__(		self   :  Tuple							,						UpperCamelCase_   :  Tuple   ):





           '''simple docstring'''

           return self.dataset[f"""{self.prefix}{key}"""]







    def __iter__(		self   :  int   ):





           '''simple docstring'''

           return iter([key for key in self.dataset if key.startswith(self.prefix   )]   )







    def __len__(		self   :  Optional[Any]   ):





           '''simple docstring'''

           return len(self.dataset   )



class 					UpperCamelCase__   (							a_):







    """simple docstring"""







    def __init__(		self   :  str							,						UpperCamelCase_   :  Dict[str, torch.Tensor] = None							,						UpperCamelCase_   :  Optional[Union[str, os.PathLike]] = None							,						UpperCamelCase_   :  Mapping = None							,						UpperCamelCase_   :  Optional[Any]=None							,						):





           '''simple docstring'''

           if state_dict is None and save_folder is None:
                  raise ValueError('Need either a `state_dict` or a `save_folder` containing offloaded weights.'   )

           __magic_name__										=						{} if state_dict is None else state_dict
           __magic_name__										=						save_folder
           if index is None and save_folder is not None:
                  with open(os.path.join(UpperCamelCase_							,						'index.json'   )   ) as f:
                         __magic_name__										=						json.load(UpperCamelCase_   )
           __magic_name__										=						{} if index is None else index
           __magic_name__										=						list(self.state_dict.keys()   )
           self.all_keys.extend([key for key in self.index if key not in self.all_keys]   )
           __magic_name__										=						device







    def __getitem__(		self   :  Dict							,						UpperCamelCase_   :  str   ):





           '''simple docstring'''

           if key in self.state_dict:
                  return self.state_dict[key]
           __magic_name__										=						self.index[key]
           if weight_info.get('safetensors_file'   ) is not None:
                  if not is_safetensors_available():
                         raise ImportError('These offloaded weights require the use of safetensors: `pip install safetensors`.'   )

                  from safetensors import safe_open

                  __magic_name__										=						'cpu' if self.device is None else self.device
                  with safe_open(weight_info['safetensors_file']							,						framework='pt'							,						device=UpperCamelCase_   ) as f:
                         __magic_name__										=						f.get_tensor(weight_info.get('weight_name'							,						UpperCamelCase_   )   )

                  if "dtype" in weight_info:
                         return tensor.to(getattr(UpperCamelCase_							,						weight_info['dtype']   )   )
                  else:
                         return tensor

           __magic_name__										=						os.path.join(self.save_folder							,						f"""{key}.dat"""   )
           return load_offloaded_weight(UpperCamelCase_							,						UpperCamelCase_   )







    def __iter__(		self   :  Any   ):





           '''simple docstring'''

           return iter(self.all_keys   )







    def __len__(		self   :  int   ):





           '''simple docstring'''

           return len(self.all_keys   )


def        A	(							__snake_case:   Dict[str, torch.Tensor]   ,    __snake_case:   List[str]     )						->		Tuple:

       """simple docstring"""





       __magic_name__										=						{}
       for module_name in submodule_names:
              # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
              # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
              result.update(
                  {
                      key: param
                      for key, param in state_dict.items()
                      if key == module_name or key.startswith(module_name + '.'     )
                  }     )
       return result







"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case   :   int    		=		logging.get_logger(__name__)

snake_case   :   List[str]    		=		{
    """uw-madison/nystromformer-512""": """https://huggingface.co/uw-madison/nystromformer-512/resolve/main/config.json""",
    # See all Nystromformer models at https://huggingface.co/models?filter=nystromformer
}



class 					UpperCamelCase__   (							a_):







    """simple docstring"""
    __UpperCAmelCase											=   """nystromformer"""







    def __init__(		self   :  Dict							,						UpperCamelCase_   :  Optional[Any]=3_0_0_0_0							,						UpperCamelCase_   :  Dict=7_6_8							,						UpperCamelCase_   :  Tuple=1_2							,						UpperCamelCase_   :  Any=1_2							,						UpperCamelCase_   :  List[str]=3_0_7_2							,						UpperCamelCase_   :  Tuple="gelu_new"							,						UpperCamelCase_   :  Optional[Any]=0.1							,						UpperCamelCase_   :  str=0.1							,						UpperCamelCase_   :  Tuple=5_1_0							,						UpperCamelCase_   :  Optional[int]=2							,						UpperCamelCase_   :  Optional[int]=6_4							,						UpperCamelCase_   :  Tuple=6_4							,						UpperCamelCase_   :  Optional[int]=6_5							,						UpperCamelCase_   :  Optional[int]=False							,						UpperCamelCase_   :  List[str]=0.02							,						UpperCamelCase_   :  List[Any]=1e-5							,						UpperCamelCase_   :  Union[str, Any]=1							,						UpperCamelCase_   :  Tuple=0							,						UpperCamelCase_   :  List[Any]=2							,						**UpperCamelCase_   :  str							,						):





           '''simple docstring'''

           __magic_name__										=						vocab_size
           __magic_name__										=						max_position_embeddings
           __magic_name__										=						hidden_size
           __magic_name__										=						num_hidden_layers
           __magic_name__										=						num_attention_heads
           __magic_name__										=						intermediate_size
           __magic_name__										=						hidden_act
           __magic_name__										=						hidden_dropout_prob
           __magic_name__										=						attention_probs_dropout_prob
           __magic_name__										=						initializer_range
           __magic_name__										=						type_vocab_size
           __magic_name__										=						segment_means_seq_len
           __magic_name__										=						num_landmarks
           __magic_name__										=						conv_kernel_size
           __magic_name__										=						inv_coeff_init_option
           __magic_name__										=						layer_norm_eps
           super().__init__(pad_token_id=UpperCamelCase_							,						bos_token_id=UpperCamelCase_							,						eos_token_id=UpperCamelCase_							,						**UpperCamelCase_   )