


'''simple docstring'''

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.test_utils import execute_subprocess_async





def __snake_case					(      UpperCAmelCase_						:						Tuple=None		):
 if subparsers is not None:
  lowerCamelCase_      =  subparsers.add_parser("test"		)
 else:
  lowerCamelCase_      =  argparse.ArgumentParser("Accelerate test command"		)

 parser.add_argument(
     "--config_file"					,				default=UpperCAmelCase_					,				help=(
         "The path to use to store the config file. Will default to a file named default_config.yaml in the cache "
         "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have "
         "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed "
         "with 'huggingface'."
     )					,				)

 if subparsers is not None:
  parser.set_defaults(func=UpperCAmelCase_		)
 return parser





def __snake_case					(      UpperCAmelCase_						:						int		):
 lowerCamelCase_      =  os.path.sep.join(__file__.split(os.path.sep		)[:-2] + ["test_utils", "scripts", "test_script.py"]		)

 if args.config_file is None:
  lowerCamelCase_      =  script_name
 else:
  lowerCamelCase_      =  F'''--config_file={args.config_file} {script_name}'''

 lowerCamelCase_      =  ["accelerate-launch"] + test_args.split()
 lowerCamelCase_      =  execute_subprocess_async(UpperCAmelCase_					,				env=os.environ.copy()		)
 if result.returncode == 0:
  print("Test is a success! You are ready for your distributed training!"		)





def __snake_case					(      ):
 lowerCamelCase_      =  test_command_parser()
 lowerCamelCase_      =  parser.parse_args()
 test_command(UpperCAmelCase_		)


if __name__ == "__main__":
     main()










'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__    :      List[Any]						=    logging.get_logger(__name__)

lowercase__    :      str						=    {}


class  SCREAMING_SNAKE_CASE    (a__							):
     lowerCAmelCase					=       '''llama'''
     lowerCAmelCase					=       ['''past_key_values''']


     def __init__(						self						,	_UpperCAmelCase=3_2000						,	_UpperCAmelCase=4096						,	_UpperCAmelCase=1_1008						,	_UpperCAmelCase=32						,	_UpperCAmelCase=32						,	_UpperCAmelCase=None						,	_UpperCAmelCase="silu"						,	_UpperCAmelCase=2048						,	_UpperCAmelCase=0.02						,	_UpperCAmelCase=1e-6						,	_UpperCAmelCase=True						,	_UpperCAmelCase=0						,	_UpperCAmelCase=1						,	_UpperCAmelCase=2						,	_UpperCAmelCase=1						,	_UpperCAmelCase=False						,	_UpperCAmelCase=None						,	**_UpperCAmelCase						,	):

          '''simple docstring'''






          __A						: Union[str, Any]      =       vocab_size
          __A						: Union[str, Any]      =       max_position_embeddings
          __A						: Any      =       hidden_size
          __A						: Optional[Any]      =       intermediate_size
          __A						: str      =       num_hidden_layers
          __A						: Optional[Any]      =       num_attention_heads

          # for backward compatibility
          if num_key_value_heads is None:
               __A						: List[Any]      =       num_attention_heads

          __A						: int      =       num_key_value_heads
          __A						: List[Any]      =       hidden_act
          __A						: Union[str, Any]      =       initializer_range
          __A						: List[Any]      =       rms_norm_eps
          __A						: Any      =       pretraining_tp
          __A						: Optional[Any]      =       use_cache
          __A						: Dict      =       rope_scaling
          self._rope_scaling_validation()

          super().__init__(
              pad_token_id=_UpperCAmelCase						,	bos_token_id=_UpperCAmelCase						,	eos_token_id=_UpperCAmelCase						,	tie_word_embeddings=_UpperCAmelCase						,	**_UpperCAmelCase						,	)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if self.rope_scaling is None:
               return

          if not isinstance(self.rope_scaling						,	_UpperCAmelCase) or len(self.rope_scaling) != 2:
               raise ValueError(
                   '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                   F'got {self.rope_scaling}')
          __A						: Optional[Any]      =       self.rope_scaling.get('type'						,	_UpperCAmelCase)
          __A						: Tuple      =       self.rope_scaling.get('factor'						,	_UpperCAmelCase)
          if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
               raise ValueError(
                   F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}')
          if rope_scaling_factor is None or not isinstance(_UpperCAmelCase						,	_UpperCAmelCase) or rope_scaling_factor <= 1.0:
               raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}')