


"""simple docstring"""


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


a_											=     logging.get_logger(__name__)

a_											=     {
    "gpt2": "https://huggingface.co/gpt2/resolve/main/config.json",
    "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/config.json",
    "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/config.json",
    "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/config.json",
    "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/config.json",
}
class snake_case   (	lowercase_):
		__UpperCamelCase							     =      'gpt2'
		__UpperCamelCase							     =      ['past_key_values']
		__UpperCamelCase							     =      {
		    'hidden_size': 'n_embd',
		    'max_position_embeddings': 'n_positions',
		    'num_attention_heads': 'n_head',
		    'num_hidden_layers': 'n_layer',
		}







		def __init__(       self   :							Optional[Any]     , a__   :							Optional[Any]=5_02_57     , a__   :							List[Any]=10_24     , a__   :							Any=7_68     , a__   :							Optional[int]=12     , a__   :							Union[str, Any]=12     , a__   :							Union[str, Any]=None     , a__   :							int="gelu_new"     , a__   :							Optional[int]=0.1     , a__   :							Union[str, Any]=0.1     , a__   :							List[Any]=0.1     , a__   :							int=1E-5     , a__   :							str=0.0_2     , a__   :							Dict="cls_index"     , a__   :							Optional[Any]=True     , a__   :							Union[str, Any]=None     , a__   :							Union[str, Any]=True     , a__   :							List[Any]=0.1     , a__   :							Any=True     , a__   :							str=True     , a__   :							Union[str, Any]=5_02_56     , a__   :							Union[str, Any]=5_02_56     , a__   :							Any=False     , a__   :							int=False     , **a__   :							Tuple     , )							->    Any:





						'''simple docstring'''







						_A       	=      vocab_size
						_A       	=      n_positions
						_A       	=      n_embd
						_A       	=      n_layer
						_A       	=      n_head
						_A       	=      n_inner
						_A       	=      activation_function
						_A       	=      resid_pdrop
						_A       	=      embd_pdrop
						_A       	=      attn_pdrop
						_A       	=      layer_norm_epsilon
						_A       	=      initializer_range
						_A       	=      summary_type
						_A       	=      summary_use_proj
						_A       	=      summary_activation
						_A       	=      summary_first_dropout
						_A       	=      summary_proj_to_labels
						_A       	=      scale_attn_weights
						_A       	=      use_cache
						_A       	=      scale_attn_by_inverse_layer_idx
						_A       	=      reorder_and_upcast_attn

						_A       	=      bos_token_id
						_A       	=      eos_token_id

						super().__init__(bos_token_id=a__     , eos_token_id=a__     , **a__		)





class snake_case   (	lowercase_):







		def __init__(       self   :							Tuple     , a__   :							Dict     , a__   :							Tuple = "default"     , a__   :							Any = None     , a__   :							Dict = False     , )							->    List[str]:





						'''simple docstring'''







						super().__init__(a__     , task=a__     , patching_specs=a__     , use_past=a__		)
						if not getattr(self._config     , "pad_token_id"     , a__		):
										# TODO: how to do that better?
										_A       	=      0







		@property
		def 			a_			(       self   :							Union[str, Any]		)							->    List[Any]:





						'''simple docstring'''







						_A       	=      OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}		)
						if self.use_past:
										self.fill_with_past_key_values_(a__     , direction="inputs"		)
										_A       	=      {0: "batch", 1: "past_sequence + sequence"}
						else:
										_A       	=      {0: "batch", 1: "sequence"}

						return common_inputs







		@property
		def 			a_			(       self   :							int		)							->    List[str]:





						'''simple docstring'''







						return self._config.n_layer







		@property
		def 			a_			(       self   :							Optional[int]		)							->    Any:





						'''simple docstring'''







						return self._config.n_head







		def 			a_			(       self   :							List[str]     , a__   :							List[str]     , a__   :							Optional[Any] = -1     , a__   :							List[Any] = -1     , a__   :							Optional[Any] = False     , a__   :							int = None     , )							->    Optional[Any]:





						'''simple docstring'''







						_A       	=      super(a__     , self		).generate_dummy_inputs(
						    a__     , batch_size=a__     , seq_length=a__     , is_pair=a__     , framework=a__		)

						# We need to order the input in the way they appears in the forward()
						_A       	=      OrderedDict({"input_ids": common_inputs["input_ids"]}		)

						# Need to add the past_keys
						if self.use_past:
										if not is_torch_available():
														raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."		)
										else:
														import torch

														_A	,		_A       	=      common_inputs["input_ids"].shape
														# Not using the same length for past_key_values
														_A       	=      seqlen + 2
														_A       	=      (
														    batch,
														    self.num_attention_heads,
														    past_key_values_length,
														    self._config.hidden_size // self.num_attention_heads,
														)
														_A       	=      [
														    (torch.zeros(a__		), torch.zeros(a__		)) for _ in range(self.num_layers		)
														]

						_A       	=      common_inputs["attention_mask"]
						if self.use_past:
										_A       	=      ordered_inputs["attention_mask"].dtype
										_A       	=      torch.cat(
										    [ordered_inputs["attention_mask"], torch.ones(a__     , a__     , dtype=a__		)]     , dim=1		)

						return ordered_inputs







		@property
		def 			a_			(       self   :							int		)							->    int:





						'''simple docstring'''







						return 13

import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
  import torch_xla.core.xla_model as xm  # noqa: F401

  _UpperCAmelCase    			=					True
except ImportError:
  _UpperCAmelCase    			=					False


# Cache this result has it's a C FFI call which can be pretty time-consuming
_UpperCAmelCase    			=					torch.distributed.is_available()


def 		_lowerCamelCase       (   _a  ):
  """simple docstring"""




  _lowerCamelCase      =						importlib.util.find_spec(_a  ) is not None
  if package_exists:
    try:
      _lowerCamelCase      =						importlib.metadata.metadata(_a  )
      return True
    except importlib.metadata.PackageNotFoundError:
      return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _torch_distributed_available


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  try:
    pass
  except ImportError:
    print(
        '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
        ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
        ''' Bindings for PyTorch*.'''  )
  return (
      importlib.util.find_spec('''torch_ccl'''  ) is not None
      or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''  ) is not None
  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return importlib.metadata.version('''oneccl_bind_pt'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''transformer_engine'''  )


@lru_cache
def 		_lowerCamelCase       (   _a=True  ):
  """simple docstring"""




  if torch.cuda.is_available():
    return False
  if _tpu_available and check_device:
    try:
      # Will raise a RuntimeError if no XLA configuration is found
      _lowerCamelCase      =						xm.xla_device()
      return True
    except RuntimeError:
      return False
  return _tpu_available


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''deepspeed'''  )


def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if is_tpu_available():
    return not ignore_tpu
  if torch.cuda.is_available():
    return torch.cuda.is_bfaa_supported()
  if is_npu_available():
    return False
  return True


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  _lowerCamelCase      =						_is_package_available('''bitsandbytes'''  )
  if package_exists:
    _lowerCamelCase      =						version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
    return compare_versions(_a			,   '''>='''			,   '''0.39.0'''  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  _lowerCamelCase      =						_is_package_available('''bitsandbytes'''  )
  if package_exists:
    _lowerCamelCase      =						version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
    return compare_versions(_a			,   '''>='''			,   '''0.37.2'''  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''bitsandbytes'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''			,   '''False'''  )  ) == 1:
    _lowerCamelCase      =						importlib.util.find_spec('''megatron'''  ) is not None
    if package_exists:
      try:
        _lowerCamelCase      =						parse(importlib.metadata.version('''megatron-lm'''  )  )
        return compare_versions(_a			,   '''>='''			,   '''2.2.0'''  )
      except Exception as e:
        warnings.warn(F'''Parse Megatron version failed. Exception:{e}'''  )
        return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''safetensors'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''transformers'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''datasets'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''aim'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''tensorboard'''  ) or _is_package_available('''tensorboardX'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''wandb'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''comet_ml'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''boto3'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  if _is_package_available('''rich'''  ):
    if "ACCELERATE_DISABLE_RICH" in os.environ:
      warnings.warn(
          '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''  )
      return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''			,   _a  )
    return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''			,   _a  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''sagemaker'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''tqdm'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''mlflow'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return is_torch_version('''>='''			,   '''1.12'''  ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  def get_major_and_minor_from_version(_a  ):
    return str(version.parse(_a  ).major  ) + "." + str(version.parse(_a  ).minor  )

  _lowerCamelCase      =						importlib.metadata.version('''torch'''  )
  if importlib.util.find_spec('''intel_extension_for_pytorch'''  ) is None:
    return False
  _lowerCamelCase      =						'''N/A'''
  try:
    _lowerCamelCase      =						importlib.metadata.version('''intel_extension_for_pytorch'''  )
  except importlib.metadata.PackageNotFoundError:
    return False
  _lowerCamelCase      =						get_major_and_minor_from_version(_torch_version  )
  _lowerCamelCase      =						get_major_and_minor_from_version(_ipex_version  )
  if torch_major_and_minor != ipex_major_and_minor:
    warnings.warn(
        F'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
        F''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''  )
    return False
  return True


@lru_cache
def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if importlib.util.find_spec('''torch'''  ) is None or importlib.util.find_spec('''torch_npu'''  ) is None:
    return False

  import torch
  import torch_npu  # noqa: F401

  if check_device:
    try:
      # Will raise a RuntimeError if no NPU is found
      _lowerCamelCase      =						torch.npu.device_count()
      return torch.npu.is_available()
    except RuntimeError:
      return False
  return hasattr(_a			,   '''npu'''  ) and torch.npu.is_available()






@lru_cache
def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if not parse_flag_from_env('''ACCELERATE_USE_XPU'''			,   default=_a  ):
    return False
  if is_ipex_available():
    import torch

    if is_torch_version('''<='''			,   '''1.12'''  ):
      return False
  else:
    return False

  import intel_extension_for_pytorch  # noqa: F401

  if check_device:
    try:
      # Will raise a RuntimeError if no XPU  is found
      _lowerCamelCase      =						torch.xpu.device_count()
      return torch.xpu.is_available()
    except RuntimeError:
      return False
  return hasattr(_a			,   '''xpu'''  ) and torch.xpu.is_available()




