






import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment




def      lowercase__      (     __snake_case    :							Optional[int]     ,       __snake_case    :							Dict=()     ,       __snake_case    :							Dict=None     ,       __snake_case    :							Dict="no"     ,       __snake_case    :							Any="29500"  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						False
					UpperCAmelCase_    :      Any            =						False
					if any(key.startswith('KAGGLE'  ) for key in os.environ.keys()  ):
										UpperCAmelCase_    :      Dict            =						True
					elif "IPython" in sys.modules:
										UpperCAmelCase_    :      int            =						'google.colab' in str(sys.modules['IPython'].get_ipython()  )

					try:
										UpperCAmelCase_    :      Optional[int]            =						PrecisionType(mixed_precision.lower()  )
					except ValueError:
										raise ValueError(
										    F"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."  )

					if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'     ,       _snake_case  ) is not None):
										# TPU launch
										import torch_xla.distributed.xla_multiprocessing as xmp

										if len(AcceleratorState._shared_state  ) > 0:
															raise ValueError(
															    'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
															    'your training function. Restart your notebook and make sure no cells initializes an '
															    '`Accelerator`.'  )
										if num_processes is None:
															UpperCAmelCase_    :      str            =						8

										UpperCAmelCase_    :      str            =						PrepareForLaunch(_snake_case     ,       distributed_type='TPU'  )
										print(F"Launching a training on {num_processes} TPU cores."  )
										xmp.spawn(_snake_case     ,       args=_snake_case     ,       nprocs=_snake_case     ,       start_method='fork'  )
					elif in_colab:
										# No need for a distributed launch otherwise as it's either CPU or one GPU.
										if torch.cuda.is_available():
															print('Launching training on one GPU.'  )
										else:
															print('Launching training on one CPU.'  )
										function(*_snake_case  )
					else:
										if num_processes is None:
															raise ValueError(
															    'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'  )

										if num_processes > 1:
															# Multi-GPU launch
															from torch.multiprocessing import start_processes
															from torch.multiprocessing.spawn import ProcessRaisedException

															if len(AcceleratorState._shared_state  ) > 0:
																				raise ValueError(
																				    'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
																				    'inside your training function. Restart your notebook and make sure no cells initializes an '
																				    '`Accelerator`.'  )

															if torch.cuda.is_initialized():
																				raise ValueError(
																				    'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
																				    'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
																				    'function.'  )

															# torch.distributed will expect a few environment variable to be here. We set the ones common to each
															# process here (the other ones will be set be the launcher).
															with patch_environment(
															    world_size=_snake_case     ,       master_addr='127.0.01'     ,       master_port=_snake_case     ,       mixed_precision=_snake_case  ):
																				UpperCAmelCase_    :      Optional[int]            =						PrepareForLaunch(_snake_case     ,       distributed_type='MULTI_GPU'  )
																				print(F"Launching training on {num_processes} GPUs."  )
																				try:
																									start_processes(_snake_case     ,       args=_snake_case     ,       nprocs=_snake_case     ,       start_method='fork'  )
																				except ProcessRaisedException as e:
																									if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																														raise RuntimeError(
																														    'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
																														    'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
																														    'Please review your imports and test them when running the `notebook_launcher()` to identify '
																														    'which one is problematic.'  ) from e

										else:
															# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
															if is_mps_available():
																				UpperCAmelCase_    :      str            =						'1'
																				print('Launching training on MPS.'  )
															elif torch.cuda.is_available():
																				print('Launching training on one GPU.'  )
															else:
																				print('Launching training on CPU.'  )
															function(*_snake_case  )






def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							List[str]=()     ,       __snake_case    :							Any=2  ):


					'''simple docstring'''


					from torch.multiprocessing import start_processes

					with tempfile.NamedTemporaryFile() as tmp_file:
										# torch.distributed will expect a few environment variable to be here. We set the ones common to each
										# process here (the other ones will be set be the launcher).
										with patch_environment(
										    world_size=_snake_case     ,       master_addr='127.0.01'     ,       master_port='29500'     ,       accelerate_mixed_precision='no'     ,       accelerate_debug_rdv_file=tmp_file.name     ,       accelerate_use_cpu='yes'     ,       ):
															UpperCAmelCase_    :      List[Any]            =						PrepareForLaunch(_snake_case     ,       debug=_snake_case  )
															start_processes(_snake_case     ,       args=_snake_case     ,       nprocs=_snake_case     ,       start_method='fork'  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase					:			Tuple  				=			logging.get_logger(__name__)

_lowerCAmelCase					:			Union[str, Any]  				=			{"ctrl": "https://huggingface.co/ctrl/resolve/main/config.json"}






class 	__magic_name__    (			lowerCAmelCase_      ):
			SCREAMING_SNAKE_CASE							   =						'ctrl'
			SCREAMING_SNAKE_CASE							   =						['past_key_values']
			SCREAMING_SNAKE_CASE							   =						{
			    'max_position_embeddings': 'n_positions',
			    'hidden_size': 'n_embd',
			    'num_attention_heads': 'n_head',
			    'num_hidden_layers': 'n_layer',
			}





			def __init__(		self					,  __snake_case=24_6534					,  __snake_case=256					,  __snake_case=1280					,  __snake_case=8192					,  __snake_case=48					,  __snake_case=16					,  __snake_case=0.1					,  __snake_case=0.1					,  __snake_case=1e-6					,  __snake_case=0.02					,  __snake_case=True					,  **__snake_case					,  )   -> Union[str, Any]:




						'''simple docstring'''
						__a        =vocab_size
						__a        =n_positions
						__a        =n_embd
						__a        =n_layer
						__a        =n_head
						__a        =dff
						__a        =resid_pdrop
						__a        =embd_pdrop
						__a        =layer_norm_epsilon
						__a        =initializer_range

						__a        =use_cache

						super().__init__(**__snake_case					)






