code: 
  
  
  import json
  import os
  from collections.abc import Mapping
  from typing import Dict, List, Optional, Union
  
  import numpy as np
  import torch
  
  from .imports import is_safetensors_available
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__     ,				lowercase__     ,				lowercase__=None			)		->			int:
  
    '''simple docstring'''
  
  
  
  
  
  
  
    __lowercase=    None
    # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
    if str(weight.dtype			) == "torch.bfloat16":
      # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
      __lowercase=    weight.view(torch.intaa			)
      __lowercase=    'bfloat16'
    __lowercase=    weight.cpu().numpy()
    __lowercase=    os.path.join(lowercase__     ,				F'{weight_name}.dat'			)
    if index is not None:
      if dtype is None:
        __lowercase=    str(array.dtype			)
      __lowercase=    {'dtype': dtype, 'shape': list(array.shape			)}
    if array.ndim == 0:
      __lowercase=    array[None]
    __lowercase=    np.memmap(lowercase__     ,				dtype=array.dtype     ,				mode='w+'     ,				shape=array.shape			)
    __lowercase=    array[:]
    file_array.flush()
    return index
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			List[str]:
  
    '''simple docstring'''
  
  
  
  
  
  
  
    __lowercase=    tuple(weight_info['shape']			)
    if shape == ():
      # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
      __lowercase=    (1,)
  
    __lowercase=    weight_info['dtype']
    if dtype == "bfloat16":
      # NumPy does not support bfloat16 so this was saved as a int16
      __lowercase=    'int16'
  
    __lowercase=    np.memmap(lowercase__     ,				dtype=lowercase__     ,				shape=lowercase__     ,				mode='r'			)
  
    if len(weight_info['shape']			) == 0:
      __lowercase=    weight[0]
    __lowercase=    torch.tensor(lowercase__			)
    if weight_info["dtype"] == "bfloat16":
      __lowercase=    weight.view(torch.bfloataa			)
  
    return weight
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			Optional[Any]:
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if index is None or len(lowercase__			) == 0:
      # Nothing to save
      return
  
    __lowercase=    os.path.join(lowercase__     ,				'index.json'			)
    if os.path.isfile(lowercase__			):
      with open(lowercase__     ,				'r'     ,				encoding='utf-8'			) as f:
        __lowercase=    json.load(lowercase__			)
    else:
      __lowercase=    {}
    current_index.update(lowercase__			)
  
    with open(lowercase__     ,				'w'     ,				encoding='utf-8'			) as f:
      json.dump(lowercase__     ,				lowercase__     ,				indent=2			)
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			int:
  
    '''simple docstring'''
  
  
  
  
  
  
  
    os.makedirs(lowercase__     ,				exist_ok=lowercase__			)
    __lowercase=    {}
    for name, parameter in state_dict.items():
      __lowercase=    offload_weight(lowercase__     ,				lowercase__     ,				lowercase__     ,				index=lowercase__			)
  
    # Update index
    save_offload_index(lowercase__     ,				lowercase__			)
  
  
  
  
  
  class    A					(     A_				):
  
  
         def __init__(self   ,       lowerCAmelCase   ,       lowerCAmelCase      ):
           __lowercase=    dataset
           __lowercase=    prefix
  
  
         def __getitem__(self   ,       lowerCAmelCase      ):
           return self.dataset[f'{self.prefix}{key}']
  
  
         def __iter__(self      ):
           return iter([key for key in self.dataset if key.startswith(self.prefix      )]      )
  
  
  
  
  
  
  
         def __len__(self      ):
           return len(self.dataset      )
  
  
  
  
  
  class    A					(     A_				):
  
  
         def __init__(self   ,       lowerCAmelCase = None   ,       lowerCAmelCase = None   ,       lowerCAmelCase = None   ,       lowerCAmelCase=None   ,       ):
           if state_dict is None and save_folder is None:
             raise ValueError('Need either a `state_dict` or a `save_folder` containing offloaded weights.'      )
  
           __lowercase=    {} if state_dict is None else state_dict
           __lowercase=    save_folder
           if index is None and save_folder is not None:
             with open(os.path.join(lowerCAmelCase   ,       'index.json'      )      ) as f:
               __lowercase=    json.load(lowerCAmelCase      )
           __lowercase=    {} if index is None else index
           __lowercase=    list(self.state_dict.keys()      )
           self.all_keys.extend([key for key in self.index if key not in self.all_keys]      )
           __lowercase=    device
  
  
         def __getitem__(self   ,       lowerCAmelCase      ):
           # State dict gets priority
           if key in self.state_dict:
             return self.state_dict[key]
           __lowercase=    self.index[key]
           if weight_info.get('safetensors_file'      ) is not None:
             if not is_safetensors_available():
               raise ImportError('These offloaded weights require the use of safetensors: `pip install safetensors`.'      )
  
             from safetensors import safe_open
  
             __lowercase=    'cpu' if self.device is None else self.device
             with safe_open(weight_info['safetensors_file']   ,       framework='pt'   ,       device=lowerCAmelCase      ) as f:
               __lowercase=    f.get_tensor(weight_info.get('weight_name'   ,       lowerCAmelCase      )      )
  
             if "dtype" in weight_info:
               return tensor.to(getattr(lowerCAmelCase   ,       weight_info['dtype']      )      )
             else:
               return tensor
  
           __lowercase=    os.path.join(self.save_folder   ,       f'{key}.dat'      )
           return load_offloaded_weight(lowerCAmelCase   ,       lowerCAmelCase      )
  
  
         def __iter__(self      ):
           return iter(self.all_keys      )
  
  
  
  
  
  
  
         def __len__(self      ):
           return len(self.all_keys      )
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			str:
  
    '''simple docstring'''
  
  
  
  
  
  
  
    __lowercase=    {}
    for module_name in submodule_names:
      # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
      # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
      result.update(
          {
              key: param
              for key, param in state_dict.items()
              if key == module_name or key.startswith(module_name + '.'			)
          }			)
    return result
  
  
  
  
  
  
  
style_context: 
  
  
  from typing import Dict, List, Optional, Tuple
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  lowerCAmelCase					 =	logging.get_logger(__name__)
  
  
  
  
  
  class    A					(     A_				):
  							UpperCamelCase_     :							Any =['''input_ids''', '''attention_mask''']
  
  
  							def __init__(self   ,       lowerCAmelCase="[PAD]"   ,       lowerCAmelCase="[BOS]"   ,       lowerCAmelCase="[EOS]"   ,       lowerCAmelCase="[MASK]"   ,       lowerCAmelCase="[CLS]"   ,       lowerCAmelCase="[SEP]"   ,       lowerCAmelCase=2_0_4_8   ,       **lowerCAmelCase   ,       ):
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else pad_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else bos_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else eos_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else mask_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else cls_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else sep_token
  
  									super().__init__(
  									    pad_token=lowerCAmelCase   ,       bos_token=lowerCAmelCase   ,       eos_token=lowerCAmelCase   ,       mask_token=lowerCAmelCase   ,       cls_token=lowerCAmelCase   ,       sep_token=lowerCAmelCase   ,       model_max_length=lowerCAmelCase   ,       **lowerCAmelCase   ,       )
  
  									__lowercase=    2**8  # utf is 8 bits
  
  									# define special tokens dict
  									__lowercase=    {
  									    self.pad_token: 0,
  									    self.bos_token: 1,
  									    self.eos_token: 2,
  									    self.mask_token: 3,
  									    self.cls_token: 4,
  									    self.sep_token: 5,
  									}
  									__lowercase=    len(self.special_tokens_encoder      )
  									__lowercase=    {v: k for k, v in self.special_tokens_encoder.items()}
  
  
  							def     _A (self      ):
  									__lowercase=    self.special_tokens_encoder.copy()
  									vocab.update(self.added_tokens_encoder      )
  									for i in range(self._utf_vocab_size      ):
  											__lowercase=    chr(lowerCAmelCase      )
  											__lowercase=    i + len(self.special_tokens_encoder      )
  									return vocab
  
  
  							@property
  							def     _A (self      ):
  									return self._utf_vocab_size + self._num_special_tokens
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None   ,       lowerCAmelCase = False      ):
  									if already_has_special_tokens:
  											return super().get_special_tokens_mask(
  											    token_ids_a=lowerCAmelCase   ,       token_ids_a=lowerCAmelCase   ,       already_has_special_tokens=lowerCAmelCase      )
  
  									# normal case: some special tokens
  									if token_ids_a is None:
  											return [1] + [0] * len(lowerCAmelCase      ) + [1]
  									return [1] + ([0] * len(lowerCAmelCase      )) + [1] + ([0] * len(lowerCAmelCase      )) + [1]
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
  									if token_ids_a is None:
  											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  									else:
  											return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									__lowercase=    [chr(lowerCAmelCase      ) for i in text.encode('utf-8'      )]
  									return tokens
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									if token in self.special_tokens_encoder:
  											__lowercase=    self.special_tokens_encoder[token]
  									elif token in self.added_tokens_encoder:
  											__lowercase=    self.added_tokens_encoder[token]
  									elif len(lowerCAmelCase      ) != 1:
  											__lowercase=    self.unk_token_id
  									else:
  											__lowercase=    ord(lowerCAmelCase      ) + self._num_special_tokens
  									return token_id
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									if index in self.special_tokens_decoder:
  											__lowercase=    self.special_tokens_decoder[index]
  									elif index in self.added_tokens_decoder:
  											__lowercase=    self.added_tokens_decoder[index]
  									else:
  											__lowercase=    chr(index - self._num_special_tokens      )
  									return token
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									__lowercase=    B''
  									for token in tokens:
  											if token in self.special_tokens_decoder:
  													__lowercase=    self.special_tokens_decoder[token].encode('utf-8'      )
  											elif token in self.added_tokens_decoder:
  													__lowercase=    self.special_tokens_decoder[token].encode('utf-8'      )
  											elif token in self.special_tokens_encoder:
  													__lowercase=    token.encode('utf-8'      )
  											elif token in self.added_tokens_encoder:
  													__lowercase=    token.encode('utf-8'      )
  											else:
  													__lowercase=    bytes([ord(lowerCAmelCase      )]      )
  											bstring += tok_string
  									__lowercase=    bstring.decode('utf-8'   ,       errors='replace'      )
  									return string
  
  
  
  
  
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
  									return ()
  
  
  
  
  
  
  