





'''simple docstring'''




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowercase__    :      Any						=    '''Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine'''






def        _lowerCAmelCase			(  )     ->     List[str]:
     __A						: str      =       _ask_options(
         'In which compute environment are you running?' ,							['This machine', 'AWS (Amazon SageMaker)'] ,							_convert_compute_environment ,							)
     if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
          __A						: Any      =       get_sagemaker_input()
     else:
          __A						: Tuple      =       get_cluster_input()
     return config






def        _lowerCAmelCase			(  __snake_case  : Optional[int]=None			)     ->     Optional[int]:
     if subparsers is not None:
          __A						: str      =       subparsers.add_parser('config' ,							description=__snake_case			)
     else:
          __A						: List[str]      =       argparse.ArgumentParser('Accelerate config command' ,							description=__snake_case			)

     parser.add_argument(
         '--config_file' ,							default=__snake_case ,							help=(
             'The path to use to store the config file. Will default to a file named default_config.yaml in the cache '
             'location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '
             'such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '
             'with \'huggingface\'.'
         ) ,							)

     if subparsers is not None:
          parser.set_defaults(func=__snake_case			)
     return parser






def        _lowerCAmelCase			(  __snake_case  : Dict			)     ->     Optional[Any]:
     __A						: Dict      =       get_user_input()
     if args.config_file is not None:
          __A						: Union[str, Any]      =       args.config_file
     else:
          if not os.path.isdir(__snake_case			):
               os.makedirs(__snake_case			)
          __A						: Union[str, Any]      =       default_yaml_config_file

     if config_file.endswith('.json'			):
          config.to_json_file(__snake_case			)
     else:
          config.to_yaml_file(__snake_case			)
     print(f'accelerate configuration saved at {config_file}'			)






def        _lowerCAmelCase			(  )     ->     Union[str, Any]:
     __A						: List[str]      =       config_command_parser()
     __A						: List[Any]      =       parser.parse_args()
     config_command(__snake_case			)


if __name__ == "__main__":
  main()






'''simple docstring'''




import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast


@dataclass
class  SCREAMING_SNAKE_CASE    (datasets.BuilderConfig							):
     lowerCAmelCase					=       None




class  SCREAMING_SNAKE_CASE    (datasets.ArrowBasedBuilder							):
     lowerCAmelCase					=       PandasConfig


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return datasets.DatasetInfo(features=self.config.features)


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          if not self.config.data_files:
               raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
          __A						: Dict      =       dl_manager.download_and_extract(self.config.data_files)
          if isinstance(_UpperCAmelCase						,	(str, list, tuple)):
               __A						: Union[str, Any]      =       data_files
               if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
                    __A						: Optional[int]      =       [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __A						: Optional[Any]      =       [dl_manager.iter_files(_UpperCAmelCase) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,	gen_kwargs={'files': files})]
          __A						: Tuple      =       []
          for split_name, files in data_files.items():
               if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
                    __A						: Dict      =       [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __A						: Optional[Any]      =       [dl_manager.iter_files(_UpperCAmelCase) for file in files]
               splits.append(datasets.SplitGenerator(name=_UpperCAmelCase						,	gen_kwargs={'files': files}))
          return splits


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          if self.config.features is not None:
               # more expensive cast to support nested features with keys in a different order
               # allows str <-> int/float or str to Audio for example
               __A						: List[str]      =       table_cast(_UpperCAmelCase						,	self.config.features.arrow_schema)
          return pa_table


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          for i, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase)):
               with open(_UpperCAmelCase						,	'rb') as f:
                    __A						: Optional[int]      =       pa.Table.from_pandas(pd.read_pickle(_UpperCAmelCase))
                    yield i, self._cast_table(_UpperCAmelCase)