
"""simple docstring"""





import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp







def 			__a  (	):
   UpperCAmelCase_      :      Tuple      =					ArgumentParser(
       description=(
           "PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"
       )						)

   # Optional arguments for the launch helper
   parser.add_argument("--num_cores",	type=__lowerCamelCase,	default=1,	help="Number of TPU cores to use (1 or 8)."						)

   # positional
   parser.add_argument(
       "training_script",	type=__lowerCamelCase,	help=(
           "The full path to the single TPU training "
           "program/script to be launched in parallel, "
           "followed by all the arguments for the "
           "training script"
       ),	)

   # rest from the training program
   parser.add_argument("training_script_args",	nargs=__lowerCamelCase						)

   return parser.parse_args()







def 			__a  (	):
   UpperCAmelCase_      :      str      =					parse_args()

   # Import training_script as a module.
   UpperCAmelCase_      :      str      =					Path(args.training_script						)
   sys.path.append(str(script_fpath.parent.resolve()						)						)
   UpperCAmelCase_      :      Optional[Any]      =					script_fpath.stem
   UpperCAmelCase_      :      Union[str, Any]      =					importlib.import_module(__lowerCamelCase						)

   # Patch sys.argv
   UpperCAmelCase_      :      Tuple      =					[args.training_script] + args.training_script_args + ["--tpu_num_cores", str(args.num_cores						)]

   xmp.spawn(mod._mp_fn,	args=(),	nprocs=args.num_cores						)


if __name__ == "__main__":
    main()




"""simple docstring"""





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_a	 =							logging.get_logger(__name__)

_a	 =							{
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}


class A_			(lowercase__   ):




  '''simple docstring'''
  SCREAMING_SNAKE_CASE__    :		Optional[int]						  =	"""bigbird_pegasus"""
  SCREAMING_SNAKE_CASE__    :		Any						  =	["""past_key_values"""]
  SCREAMING_SNAKE_CASE__    :		Tuple						  =	{
      """num_attention_heads""": """encoder_attention_heads""",
      """hidden_size""": """d_model""",
      """attention_probs_dropout_prob""": """attention_dropout""",
  }







  def __init__(	self       ,     lowercase_=9_6103       ,     lowercase_=4096       ,     lowercase_=16       ,     lowercase_=4096       ,     lowercase_=16       ,     lowercase_=16       ,     lowercase_=4096       ,     lowercase_=16       ,     lowercase_=0.0       ,     lowercase_=0.0       ,     lowercase_=True       ,     lowercase_=True       ,     lowercase_="gelu_new"       ,     lowercase_=1024       ,     lowercase_=0.1       ,     lowercase_=0.0       ,     lowercase_=0.0       ,     lowercase_=0.02       ,     lowercase_=2       ,     lowercase_=0.0       ,     lowercase_=True       ,     lowercase_=0       ,     lowercase_=2       ,     lowercase_=1       ,     lowercase_="block_sparse"       ,     lowercase_=64       ,     lowercase_=3       ,     lowercase_=False       ,     **lowercase_       ,     ):

     """simple docstring"""
     UpperCAmelCase_      :      Union[str, Any]      =					vocab_size
     UpperCAmelCase_      :      Optional[Any]      =					max_position_embeddings
     UpperCAmelCase_      :      List[str]      =					d_model
     UpperCAmelCase_      :      str      =					encoder_ffn_dim
     UpperCAmelCase_      :      Optional[Any]      =					encoder_layers
     UpperCAmelCase_      :      Optional[Any]      =					encoder_attention_heads
     UpperCAmelCase_      :      str      =					decoder_ffn_dim
     UpperCAmelCase_      :      Optional[Any]      =					decoder_layers
     UpperCAmelCase_      :      Optional[int]      =					decoder_attention_heads
     UpperCAmelCase_      :      Dict      =					dropout
     UpperCAmelCase_      :      Optional[int]      =					attention_dropout
     UpperCAmelCase_      :      Any      =					activation_dropout
     UpperCAmelCase_      :      str      =					activation_function
     UpperCAmelCase_      :      Tuple      =					init_std
     UpperCAmelCase_      :      Optional[int]      =					encoder_layerdrop
     UpperCAmelCase_      :      Union[str, Any]      =					decoder_layerdrop
     UpperCAmelCase_      :      int      =					classifier_dropout
     UpperCAmelCase_      :      Union[str, Any]      =					use_cache
     UpperCAmelCase_      :      List[str]      =					encoder_layers
     UpperCAmelCase_      :      str      =					scale_embedding  # scale factor will be sqrt(d_model) if True

     # extra config
     UpperCAmelCase_      :      List[Any]      =					attention_type
     UpperCAmelCase_      :      Any      =					block_size
     UpperCAmelCase_      :      str      =					num_random_blocks
     UpperCAmelCase_      :      Optional[Any]      =					use_bias

     super().__init__(
         pad_token_id=lowercase_       ,     bos_token_id=lowercase_       ,     eos_token_id=lowercase_       ,     is_encoder_decoder=lowercase_       ,     decoder_start_token_id=lowercase_       ,     **lowercase_       ,     )




class A_			(lowercase__   ):




  '''simple docstring'''







  @property
  def  UpperCamelCase__      (	self						):

     """simple docstring"""
     if self.task in ["default", "seq2seq-lm"]:
        UpperCAmelCase_      :      List[str]      =					OrderedDict(
            [
                ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
            ]						)

        if self.use_past:
           UpperCAmelCase_      :      List[str]      =					{0: "batch"}
           UpperCAmelCase_      :      Any      =					{0: "batch", 1: "past_decoder_sequence + sequence"}
        else:
           UpperCAmelCase_      :      List[str]      =					{0: "batch", 1: "decoder_sequence"}
           UpperCAmelCase_      :      Optional[Any]      =					{0: "batch", 1: "decoder_sequence"}

        if self.use_past:
           self.fill_with_past_key_values_(lowercase_       ,     direction="inputs"						)
     elif self.task == "causal-lm":
        # TODO: figure this case out.
        UpperCAmelCase_      :      str      =					OrderedDict(
            [
                ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
            ]						)
        if self.use_past:
           UpperCAmelCase_  ,     UpperCAmelCase_      :      Dict      =					self.num_layers
           for i in range(lowercase_						):
              UpperCAmelCase_      :      Tuple      =					{0: "batch", 2: "past_sequence + sequence"}
              UpperCAmelCase_      :      List[str]      =					{0: "batch", 2: "past_sequence + sequence"}
     else:
        UpperCAmelCase_      :      Optional[Any]      =					OrderedDict(
            [
                ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
                ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
                ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
            ]						)

     return common_inputs







  @property
  def  UpperCamelCase__      (	self						):

     """simple docstring"""
     if self.task in ["default", "seq2seq-lm"]:
        UpperCAmelCase_      :      Union[str, Any]      =					super().outputs
     else:
        UpperCAmelCase_      :      Optional[Any]      =					super(lowercase_       ,     self						).outputs
        if self.use_past:
           UpperCAmelCase_  ,     UpperCAmelCase_      :      str      =					self.num_layers
           for i in range(lowercase_						):
              UpperCAmelCase_      :      Any      =					{0: "batch", 2: "past_sequence + sequence"}
              UpperCAmelCase_      :      int      =					{0: "batch", 2: "past_sequence + sequence"}
     return common_outputs







  def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_ = -1       ,     lowercase_ = -1       ,     lowercase_ = False       ,     lowercase_ = None       ,     ):

     """simple docstring"""
     UpperCAmelCase_      :      str      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						)

     # Generate decoder inputs
     UpperCAmelCase_      :      Union[str, Any]      =					seq_length if not self.use_past else 1
     UpperCAmelCase_      :      str      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						)
     UpperCAmelCase_      :      List[Any]      =					{F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
     UpperCAmelCase_      :      Optional[Any]      =					dict(**lowercase_       ,     **lowercase_						)

     if self.use_past:
        if not is_torch_available():
           raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."						)
        else:
           import torch
        UpperCAmelCase_  ,     UpperCAmelCase_      :      Dict      =					common_inputs["input_ids"].shape
        UpperCAmelCase_      :      Union[str, Any]      =					common_inputs["decoder_input_ids"].shape[1]
        UpperCAmelCase_  ,     UpperCAmelCase_      :      Tuple      =					self.num_attention_heads
        UpperCAmelCase_      :      Tuple      =					(
            batch,
            num_encoder_attention_heads,
            encoder_seq_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )
        UpperCAmelCase_      :      Union[str, Any]      =					decoder_seq_length + 3
        UpperCAmelCase_      :      List[Any]      =					(
            batch,
            num_decoder_attention_heads,
            decoder_past_length,
            self._config.hidden_size // num_decoder_attention_heads,
        )

        UpperCAmelCase_      :      List[str]      =					torch.cat(
            [common_inputs["decoder_attention_mask"], torch.ones(lowercase_       ,     lowercase_						)]       ,     dim=1						)

        UpperCAmelCase_      :      Dict      =					[]
        # If the number of encoder and decoder layers are present in the model configuration, both are considered
        UpperCAmelCase_  ,     UpperCAmelCase_      :      Union[str, Any]      =					self.num_layers
        UpperCAmelCase_      :      List[str]      =					min(lowercase_       ,     lowercase_						)
        UpperCAmelCase_      :      Optional[int]      =					max(lowercase_       ,     lowercase_						) - min_num_layers
        UpperCAmelCase_      :      List[Any]      =					"encoder" if num_encoder_layers > num_decoder_layers else "decoder"

        for _ in range(lowercase_						):
           common_inputs["past_key_values"].append(
               (
                   torch.zeros(lowercase_						),
                   torch.zeros(lowercase_						),
                   torch.zeros(lowercase_						),
                   torch.zeros(lowercase_						),
               )						)
        # TODO: test this.
        UpperCAmelCase_      :      Optional[int]      =					encoder_shape if remaining_side_name == "encoder" else decoder_shape
        for _ in range(lowercase_       ,     lowercase_						):
           common_inputs["past_key_values"].append((torch.zeros(lowercase_						), torch.zeros(lowercase_						))						)
     return common_inputs







  def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_ = -1       ,     lowercase_ = -1       ,     lowercase_ = False       ,     lowercase_ = None       ,     ):

     """simple docstring"""
     UpperCAmelCase_      :      Optional[Any]      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						)

     if self.use_past:
        if not is_torch_available():
           raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."						)
        else:
           import torch
        UpperCAmelCase_  ,     UpperCAmelCase_      :      Tuple      =					common_inputs["input_ids"].shape
        # Not using the same length for past_key_values
        UpperCAmelCase_      :      str      =					seqlen + 2
        UpperCAmelCase_  ,     UpperCAmelCase_      :      List[Any]      =					self.num_layers
        UpperCAmelCase_  ,     UpperCAmelCase_      :      Dict      =					self.num_attention_heads
        UpperCAmelCase_      :      Optional[int]      =					(
            batch,
            num_encoder_attention_heads,
            past_key_values_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )

        UpperCAmelCase_      :      str      =					common_inputs["attention_mask"].dtype
        UpperCAmelCase_      :      Union[str, Any]      =					torch.cat(
            [common_inputs["attention_mask"], torch.ones(lowercase_       ,     lowercase_       ,     dtype=lowercase_						)]       ,     dim=1						)
        UpperCAmelCase_      :      List[str]      =					[
            (torch.zeros(lowercase_						), torch.zeros(lowercase_						)) for _ in range(lowercase_						)
        ]
     return common_inputs







  def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_ = -1       ,     lowercase_ = -1       ,     lowercase_ = False       ,     lowercase_ = None       ,     ):

     """simple docstring"""
     # Copied from OnnxConfig.generate_dummy_inputs
     # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
     # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
     UpperCAmelCase_      :      Any      =					compute_effective_axis_dimension(
         lowercase_       ,     fixed_dimension=OnnxConfig.default_fixed_batch       ,     num_token_to_add=0						)

     # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
     UpperCAmelCase_      :      List[str]      =					tokenizer.num_special_tokens_to_add(lowercase_						)
     UpperCAmelCase_      :      Union[str, Any]      =					compute_effective_axis_dimension(
         lowercase_       ,     fixed_dimension=OnnxConfig.default_fixed_sequence       ,     num_token_to_add=lowercase_						)

     # Generate dummy inputs according to compute batch and sequence
     UpperCAmelCase_      :      str      =					[" ".join([tokenizer.unk_token]						) * seq_length] * batch_size
     UpperCAmelCase_      :      int      =					dict(tokenizer(lowercase_       ,     return_tensors=lowercase_						)						)
     return common_inputs







  def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_ = -1       ,     lowercase_ = -1       ,     lowercase_ = False       ,     lowercase_ = None       ,     ):

     """simple docstring"""
     if self.task in ["default", "seq2seq-lm"]:
        UpperCAmelCase_      :      Dict      =					self._generate_dummy_inputs_for_default_and_seqaseq_lm(
            lowercase_       ,     batch_size=lowercase_       ,     seq_length=lowercase_       ,     is_pair=lowercase_       ,     framework=lowercase_						)

     elif self.task == "causal-lm":
        UpperCAmelCase_      :      List[str]      =					self._generate_dummy_inputs_for_causal_lm(
            lowercase_       ,     batch_size=lowercase_       ,     seq_length=lowercase_       ,     is_pair=lowercase_       ,     framework=lowercase_						)
     else:
        UpperCAmelCase_      :      int      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            lowercase_       ,     batch_size=lowercase_       ,     seq_length=lowercase_       ,     is_pair=lowercase_       ,     framework=lowercase_						)

     return common_inputs







  def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						):

     """simple docstring"""
     if self.task in ["default", "seq2seq-lm"]:
        UpperCAmelCase_      :      Union[str, Any]      =					super()._flatten_past_key_values_(lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						)
     else:
        UpperCAmelCase_      :      Dict      =					super(lowercase_       ,     self						)._flatten_past_key_values_(
            lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						)


