code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  from collections import OrderedDict
  from typing import Any, Mapping, Optional
  
  from ... import PreTrainedTokenizer
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
  from ...onnx.utils import compute_effective_axis_dimension
  from ...utils import TensorType, is_torch_available, logging
  
  
  _lowerCamelCase							:		Tuple							=						logging.get_logger(__name__)
  
  _lowerCamelCase							:		List[Any]							=						{
      'google/bigbird-pegasus-large-arxiv': (
          'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
      ),
      'google/bigbird-pegasus-large-pubmed': (
          'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
      ),
      'google/bigbird-pegasus-large-bigpatent': (
          'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
      ),
      # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
  }
  
  
  
  
  
  class   lowerCamelCase_							( UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				"""bigbird_pegasus"""
  					lowercase__					=				["""past_key_values"""]
  					lowercase__					=				{
  					    """num_attention_heads""": """encoder_attention_heads""",
  					    """hidden_size""": """d_model""",
  					    """attention_probs_dropout_prob""": """attention_dropout""",
  					}
  
  
  
  
  
  
  					def __init__(					self,						__UpperCamelCase=96_103,						__UpperCamelCase=4_096,						__UpperCamelCase=16,						__UpperCamelCase=4_096,						__UpperCamelCase=16,						__UpperCamelCase=16,						__UpperCamelCase=4_096,						__UpperCamelCase=16,						__UpperCamelCase=0.0,						__UpperCamelCase=0.0,						__UpperCamelCase=True,						__UpperCamelCase=True,						__UpperCamelCase="gelu_new",						__UpperCamelCase=1_024,						__UpperCamelCase=0.1,						__UpperCamelCase=0.0,						__UpperCamelCase=0.0,						__UpperCamelCase=0.02,						__UpperCamelCase=2,						__UpperCamelCase=0.0,						__UpperCamelCase=True,						__UpperCamelCase=0,						__UpperCamelCase=2,						__UpperCamelCase=1,						__UpperCamelCase="block_sparse",						__UpperCamelCase=64,						__UpperCamelCase=3,						__UpperCamelCase=False,						**__UpperCamelCase,						):
  							_snake_case			 = vocab_size
  							_snake_case			 = max_position_embeddings
  							_snake_case			 = d_model
  							_snake_case			 = encoder_ffn_dim
  							_snake_case			 = encoder_layers
  							_snake_case			 = encoder_attention_heads
  							_snake_case			 = decoder_ffn_dim
  							_snake_case			 = decoder_layers
  							_snake_case			 = decoder_attention_heads
  							_snake_case			 = dropout
  							_snake_case			 = attention_dropout
  							_snake_case			 = activation_dropout
  							_snake_case			 = activation_function
  							_snake_case			 = init_std
  							_snake_case			 = encoder_layerdrop
  							_snake_case			 = decoder_layerdrop
  							_snake_case			 = classifier_dropout
  							_snake_case			 = use_cache
  							_snake_case			 = encoder_layers
  							_snake_case			 = scale_embedding  # scale factor will be sqrt(d_model) if True
  
  							# extra config
  							_snake_case			 = attention_type
  							_snake_case			 = block_size
  							_snake_case			 = num_random_blocks
  							_snake_case			 = use_bias
  
  							super().__init__(
  							    pad_token_id=A_,						bos_token_id=A_,						eos_token_id=A_,						is_encoder_decoder=A_,						decoder_start_token_id=A_,						**A_,						)
  
  
  
  
  
  class   lowerCamelCase_							( UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					@property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							if self.task in ["default", "seq2seq-lm"]:
  									_snake_case			 = OrderedDict(
  									    [
  									        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
  									        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
  									    ]				)
  
  									if self.use_past:
  											_snake_case			 = {0: "batch"}
  											_snake_case			 = {0: "batch", 1: "past_decoder_sequence + sequence"}
  									else:
  											_snake_case			 = {0: "batch", 1: "decoder_sequence"}
  											_snake_case			 = {0: "batch", 1: "decoder_sequence"}
  
  									if self.use_past:
  											self.fill_with_past_key_values_(A_,						direction="inputs"				)
  							elif self.task == "causal-lm":
  									# TODO: figure this case out.
  									_snake_case			 = OrderedDict(
  									    [
  									        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
  									        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
  									    ]				)
  									if self.use_past:
  											_snake_case ,					_snake_case			 = self.num_layers
  											for i in range(A_				):
  													_snake_case			 = {0: "batch", 2: "past_sequence + sequence"}
  													_snake_case			 = {0: "batch", 2: "past_sequence + sequence"}
  							else:
  									_snake_case			 = OrderedDict(
  									    [
  									        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
  									        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
  									        ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
  									        ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
  									    ]				)
  
  							return common_inputs
  
  
  
  
  
  
  					@property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							if self.task in ["default", "seq2seq-lm"]:
  									_snake_case			 = super().outputs
  							else:
  									_snake_case			 = super(A_,						self				).outputs
  									if self.use_past:
  											_snake_case ,					_snake_case			 = self.num_layers
  											for i in range(A_				):
  													_snake_case			 = {0: "batch", 2: "past_sequence + sequence"}
  													_snake_case			 = {0: "batch", 2: "past_sequence + sequence"}
  							return common_outputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = -1,						__UpperCamelCase = -1,						__UpperCamelCase = False,						__UpperCamelCase = None,						):
  							_snake_case			 = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
  							    A_,						A_,						A_,						A_,						A_				)
  
  							# Generate decoder inputs
  							_snake_case			 = seq_length if not self.use_past else 1
  							_snake_case			 = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
  							    A_,						A_,						A_,						A_,						A_				)
  							_snake_case			 = {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
  							_snake_case			 = dict(**A_,						**A_				)
  
  							if self.use_past:
  									if not is_torch_available():
  											raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
  									else:
  											import torch
  									_snake_case ,					_snake_case			 = common_inputs["input_ids"].shape
  									_snake_case			 = common_inputs["decoder_input_ids"].shape[1]
  									_snake_case ,					_snake_case			 = self.num_attention_heads
  									_snake_case			 = (
  									    batch,
  									    num_encoder_attention_heads,
  									    encoder_seq_length,
  									    self._config.hidden_size // num_encoder_attention_heads,
  									)
  									_snake_case			 = decoder_seq_length + 3
  									_snake_case			 = (
  									    batch,
  									    num_decoder_attention_heads,
  									    decoder_past_length,
  									    self._config.hidden_size // num_decoder_attention_heads,
  									)
  
  									_snake_case			 = torch.cat(
  									    [common_inputs["decoder_attention_mask"], torch.ones(A_,						A_				)],						dim=1				)
  
  									_snake_case			 = []
  									# If the number of encoder and decoder layers are present in the model configuration, both are considered
  									_snake_case ,					_snake_case			 = self.num_layers
  									_snake_case			 = min(A_,						A_				)
  									_snake_case			 = max(A_,						A_				) - min_num_layers
  									_snake_case			 = "encoder" if num_encoder_layers > num_decoder_layers else "decoder"
  
  									for _ in range(A_				):
  											common_inputs["past_key_values"].append(
  											    (
  											        torch.zeros(A_				),
  											        torch.zeros(A_				),
  											        torch.zeros(A_				),
  											        torch.zeros(A_				),
  											    )				)
  									# TODO: test this.
  									_snake_case			 = encoder_shape if remaining_side_name == "encoder" else decoder_shape
  									for _ in range(A_,						A_				):
  											common_inputs["past_key_values"].append((torch.zeros(A_				), torch.zeros(A_				))				)
  							return common_inputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = -1,						__UpperCamelCase = -1,						__UpperCamelCase = False,						__UpperCamelCase = None,						):
  							_snake_case			 = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
  							    A_,						A_,						A_,						A_,						A_				)
  
  							if self.use_past:
  									if not is_torch_available():
  											raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
  									else:
  											import torch
  									_snake_case ,					_snake_case			 = common_inputs["input_ids"].shape
  									# Not using the same length for past_key_values
  									_snake_case			 = seqlen + 2
  									_snake_case ,					_snake_case			 = self.num_layers
  									_snake_case ,					_snake_case			 = self.num_attention_heads
  									_snake_case			 = (
  									    batch,
  									    num_encoder_attention_heads,
  									    past_key_values_length,
  									    self._config.hidden_size // num_encoder_attention_heads,
  									)
  
  									_snake_case			 = common_inputs["attention_mask"].dtype
  									_snake_case			 = torch.cat(
  									    [common_inputs["attention_mask"], torch.ones(A_,						A_,						dtype=A_				)],						dim=1				)
  									_snake_case			 = [
  									    (torch.zeros(A_				), torch.zeros(A_				)) for _ in range(A_				)
  									]
  							return common_inputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = -1,						__UpperCamelCase = -1,						__UpperCamelCase = False,						__UpperCamelCase = None,						):
  							# Copied from OnnxConfig.generate_dummy_inputs
  							# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
  							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
  							_snake_case			 = compute_effective_axis_dimension(
  							    A_,						fixed_dimension=OnnxConfig.default_fixed_batch,						num_token_to_add=0				)
  
  							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
  							_snake_case			 = tokenizer.num_special_tokens_to_add(A_				)
  							_snake_case			 = compute_effective_axis_dimension(
  							    A_,						fixed_dimension=OnnxConfig.default_fixed_sequence,						num_token_to_add=A_				)
  
  							# Generate dummy inputs according to compute batch and sequence
  							_snake_case			 = [" ".join([tokenizer.unk_token]				) * seq_length] * batch_size
  							_snake_case			 = dict(tokenizer(A_,						return_tensors=A_				)				)
  							return common_inputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = -1,						__UpperCamelCase = -1,						__UpperCamelCase = False,						__UpperCamelCase = None,						):
  							if self.task in ["default", "seq2seq-lm"]:
  									_snake_case			 = self._generate_dummy_inputs_for_default_and_seqaseq_lm(
  									    A_,						batch_size=A_,						seq_length=A_,						is_pair=A_,						framework=A_				)
  
  							elif self.task == "causal-lm":
  									_snake_case			 = self._generate_dummy_inputs_for_causal_lm(
  									    A_,						batch_size=A_,						seq_length=A_,						is_pair=A_,						framework=A_				)
  							else:
  									_snake_case			 = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
  									    A_,						batch_size=A_,						seq_length=A_,						is_pair=A_,						framework=A_				)
  
  							return common_inputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				):
  							if self.task in ["default", "seq2seq-lm"]:
  									_snake_case			 = super()._flatten_past_key_values_(A_,						A_,						A_,						A_				)
  							else:
  									_snake_case			 = super(A_,						self				)._flatten_past_key_values_(
  									    A_,						A_,						A_,						A_				)
  
  
  
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import importlib
  import importlib.metadata
  import os
  import warnings
  from distutils.util import strtobool
  from functools import lru_cache
  
  import torch
  from packaging import version
  from packaging.version import parse
  
  from .environment import parse_flag_from_env
  from .versions import compare_versions, is_torch_version
  
  
  try:
  						import torch_xla.core.xla_model as xm  # noqa: F401
  
  						_lowerCamelCase							:		int										=						True
  except ImportError:
  						_lowerCamelCase							:		List[str]										=						False
  
  
  # Cache this result has it's a C FFI call which can be pretty time-consuming
  _lowerCamelCase							:		List[Any]										=						torch.distributed.is_available()
  
  
  
  
  
  
  def 	_lowercase	(			lowercase_						):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		_snake_case			 = importlib.util.find_spec(lowercase_						) is not None
  		if package_exists:
  				try:
  						_snake_case			 = importlib.metadata.metadata(lowercase_						)
  						return True
  				except importlib.metadata.PackageNotFoundError:
  						return False
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _torch_distributed_available
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		try:
  				pass
  		except ImportError:
  				print(
  				    "Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not"
  				    " detected. If you see \"ValueError: Invalid backend: 'ccl'\" error, please install Intel(R) oneCCL"
  				    " Bindings for PyTorch*."						)
  		return (
  		    importlib.util.find_spec("torch_ccl"						) is not None
  		    or importlib.util.find_spec("oneccl_bindings_for_pytorch"						) is not None
  		)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return importlib.metadata.version("oneccl_bind_pt"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("transformer_engine"						)
  
  
  
  
  
  
  @lru_cache
  def 	_lowercase	(			lowercase_=True						):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if torch.cuda.is_available():
  				return False
  		if _tpu_available and check_device:
  				try:
  						# Will raise a RuntimeError if no XLA configuration is found
  						_snake_case			 = xm.xla_device()
  						return True
  				except RuntimeError:
  						return False
  		return _tpu_available
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("deepspeed"						)
  
  
  
  
  
  
  def 	_lowercase	(			lowercase_=False						):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if is_tpu_available():
  				return not ignore_tpu
  		if torch.cuda.is_available():
  				return torch.cuda.is_bfaa_supported()
  		if is_npu_available():
  				return False
  		return True
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		_snake_case			 = _is_package_available("bitsandbytes"						)
  		if package_exists:
  				_snake_case			 = version.parse(importlib.metadata.version("bitsandbytes"						)						)
  				return compare_versions(lowercase_		,							">="		,							"0.39.0"						)
  		return False
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		_snake_case			 = _is_package_available("bitsandbytes"						)
  		if package_exists:
  				_snake_case			 = version.parse(importlib.metadata.version("bitsandbytes"						)						)
  				return compare_versions(lowercase_		,							">="		,							"0.37.2"						)
  		return False
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("bitsandbytes"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if strtobool(os.environ.get("ACCELERATE_USE_MEGATRON_LM"		,							"False"						)						) == 1:
  				_snake_case			 = importlib.util.find_spec("megatron"						) is not None
  				if package_exists:
  						try:
  								_snake_case			 = parse(importlib.metadata.version("megatron-lm"						)						)
  								return compare_versions(lowercase_		,							">="		,							"2.2.0"						)
  						except Exception as e:
  								warnings.warn(f'''Parse Megatron version failed. Exception:{e}'''						)
  								return False
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("safetensors"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("transformers"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("datasets"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("aim"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("tensorboard"						) or _is_package_available("tensorboardX"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("wandb"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("comet_ml"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("boto3"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if _is_package_available("rich"						):
  				if "ACCELERATE_DISABLE_RICH" in os.environ:
  						warnings.warn(
  						    "`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`."						)
  						return not parse_flag_from_env("ACCELERATE_DISABLE_RICH"		,							lowercase_						)
  				return parse_flag_from_env("ACCELERATE_ENABLE_RICH"		,							lowercase_						)
  		return False
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("sagemaker"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("tqdm"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return _is_package_available("mlflow"						)
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		return is_torch_version(">="		,							"1.12"						) and torch.backends.mps.is_available() and torch.backends.mps.is_built()
  
  
  
  
  
  
  def 	_lowercase	(			):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def get_major_and_minor_from_version(lowercase_						):
  				return str(version.parse(lowercase_						).major						) + "." + str(version.parse(lowercase_						).minor						)
  
  		_snake_case			 = importlib.metadata.version("torch"						)
  		if importlib.util.find_spec("intel_extension_for_pytorch"						) is None:
  				return False
  		_snake_case			 = "N/A"
  		try:
  				_snake_case			 = importlib.metadata.version("intel_extension_for_pytorch"						)
  		except importlib.metadata.PackageNotFoundError:
  				return False
  		_snake_case			 = get_major_and_minor_from_version(_torch_version						)
  		_snake_case			 = get_major_and_minor_from_version(_ipex_version						)
  		if torch_major_and_minor != ipex_major_and_minor:
  				warnings.warn(
  				    f'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
  				    f''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''						)
  				return False
  		return True
  
  
  
  
  
  
  @lru_cache
  def 	_lowercase	(			lowercase_=False						):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if importlib.util.find_spec("torch"						) is None or importlib.util.find_spec("torch_npu"						) is None:
  				return False
  
  		import torch
  		import torch_npu  # noqa: F401
  
  		if check_device:
  				try:
  						# Will raise a RuntimeError if no NPU is found
  						_snake_case			 = torch.npu.device_count()
  						return torch.npu.is_available()
  				except RuntimeError:
  						return False
  		return hasattr(lowercase_		,							"npu"						) and torch.npu.is_available()
  
  
  
  
  
  
  @lru_cache
  def 	_lowercase	(			lowercase_=False						):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if not parse_flag_from_env("ACCELERATE_USE_XPU"		,							default=lowercase_						):
  				return False
  		if is_ipex_available():
  				import torch
  
  				if is_torch_version("<="		,							"1.12"						):
  						return False
  		else:
  				return False
  
  		import intel_extension_for_pytorch  # noqa: F401
  
  		if check_device:
  				try:
  						# Will raise a RuntimeError if no XPU  is found
  						_snake_case			 = torch.xpu.device_count()
  						return torch.xpu.is_available()
  				except RuntimeError:
  						return False
  		return hasattr(lowercase_		,							"xpu"						) and torch.xpu.is_available()
  
  
  
  
  
  