code: 
  
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  A_    :					List[Any]             =logging.get_logger(__name__)
  
  A_    :					Dict             ={
      """facebook/opt-125m""": """https://huggingface.co/facebook/opt-125m/blob/main/config.json""",
      """facebook/opt-350m""": """https://huggingface.co/facebook/opt-350m/blob/main/config.json""",
      """facebook/opt-1.3b""": """https://huggingface.co/facebook/opt-1.3b/blob/main/config.json""",
      """facebook/opt-2.7b""": """https://huggingface.co/facebook/opt-2.7b/blob/main/config.json""",
      """facebook/opt-6.7b""": """https://huggingface.co/facebook/opt-6.7b/blob/main/config.json""",
      """facebook/opt-13b""": """https://huggingface.co/facebook/opt-13b/blob/main/config.json""",
  }
  
  
  
  
  
  class        lowercase_				( UpperCamelCase__):
  
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  
  
  
  							snake_case_        =		'''opt'''
  							snake_case_        =		['''past_key_values''']
  
  
  							def __init__(   self			,    _UpperCAmelCase=50_272			,    _UpperCAmelCase=768			,    _UpperCAmelCase=12			,    _UpperCAmelCase=3_072			,    _UpperCAmelCase=2_048			,    _UpperCAmelCase=True			,    _UpperCAmelCase=False			,    _UpperCAmelCase=None			,    _UpperCAmelCase=0.1			,    _UpperCAmelCase=0.0			,    _UpperCAmelCase=12			,    _UpperCAmelCase="relu"			,    _UpperCAmelCase=0.0			,    _UpperCAmelCase=0.0_2			,    _UpperCAmelCase=True			,    _UpperCAmelCase=1			,    _UpperCAmelCase=2			,    _UpperCAmelCase=2			,    _UpperCAmelCase=True			,    _UpperCAmelCase=True			,    **_UpperCAmelCase			,    ):
  
  
  
  
  
  
  
  													"""simple docstring"""
  
  
  													super().__init__(
  													    pad_token_id=_UpperCAmelCase			,    bos_token_id=_UpperCAmelCase			,    eos_token_id=_UpperCAmelCase			,    **_UpperCAmelCase			,    )
  													a_					=		vocab_size
  													a_					=		max_position_embeddings
  													a_					=		num_attention_heads
  													a_					=		word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
  													a_					=		ffn_dim
  													a_					=		hidden_size
  													a_					=		num_hidden_layers
  													a_					=		dropout
  													a_					=		attention_dropout
  													a_					=		activation_function
  													a_					=		init_std
  													a_					=		layerdrop
  													a_					=		use_cache
  													a_					=		do_layer_norm_before
  													# We keep these variables at `True` for backward compatibility.
  													a_					=		enable_bias
  													a_					=		layer_norm_elementwise_affine
  
  													# Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
  													# with checkpoints that have been fine-tuned before transformers v4.20.1
  													# see https://github.com/facebookresearch/metaseq/pull/164
  													a_					=		_remove_final_layer_norm
style_context: 
  
  
  import importlib
  import sys
  from argparse import REMAINDER, ArgumentParser
  from pathlib import Path
  
  import torch_xla.distributed.xla_multiprocessing as xmp
  
  
  
  
  def   lowerCamelCase_       (					):
  
  
  
  						"""simple docstring"""
  
  
  
  
  						a_					=		ArgumentParser(
  						    description=(
  						        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
  						    )      )
  
  						# Optional arguments for the launch helper
  						parser.add_argument("""--num_cores"""       ,    type=UpperCAmelCase__       ,    default=1       ,    help="""Number of TPU cores to use (1 or 8)."""      )
  
  						# positional
  						parser.add_argument(
  						    """training_script"""       ,    type=UpperCAmelCase__       ,    help=(
  						        """The full path to the single TPU training """
  						        """program/script to be launched in parallel, """
  						        """followed by all the arguments for the """
  						        """training script"""
  						    )       ,    )
  
  						# rest from the training program
  						parser.add_argument("""training_script_args"""       ,    nargs=UpperCAmelCase__      )
  
  						return parser.parse_args()
  
  
  
  
  def   lowerCamelCase_       (					):
  
  
  
  						"""simple docstring"""
  
  
  
  
  						a_					=		parse_args()
  
  						# Import training_script as a module.
  						a_					=		Path(args.training_script      )
  						sys.path.append(str(script_fpath.parent.resolve()      )      )
  						a_					=		script_fpath.stem
  						a_					=		importlib.import_module(UpperCAmelCase__      )
  
  						# Patch sys.argv
  						a_					=		[args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores      )]
  
  						xmp.spawn(mod._mp_fn       ,    args=()       ,    nprocs=args.num_cores      )
  
  
  if __name__ == "__main__":
  	main()