


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


UpperCamelCase								=	"""Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"""


def   _SCREAMING_SNAKE_CASE	( ):
  A_      :				int		     =					_ask_options(
      '''In which compute environment are you running?''' ,			['''This machine''', '''AWS (Amazon SageMaker)'''] ,			_convert_compute_environment ,			)
  if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
    A_      :				Any		     =					get_sagemaker_input()
  else:
    A_      :				int		     =					get_cluster_input()
  return config


def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE=None       ):
  if subparsers is not None:
    A_      :				str		     =					subparsers.add_parser('''config''' ,			description=SCREAMING_SNAKE_CASE       )
  else:
    A_      :				Any		     =					argparse.ArgumentParser('''Accelerate config command''' ,			description=SCREAMING_SNAKE_CASE       )

  parser.add_argument(
      '''--config_file''' ,			default=SCREAMING_SNAKE_CASE ,			help=(
          '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
          '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
          '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
          '''with \'huggingface\'.'''
      ) ,			)

  if subparsers is not None:
    parser.set_defaults(func=SCREAMING_SNAKE_CASE       )
  return parser


def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE       ):
  A_      :				int		     =					get_user_input()
  if args.config_file is not None:
    A_      :				Union[str, Any]		     =					args.config_file
  else:
    if not os.path.isdir(SCREAMING_SNAKE_CASE       ):
      os.makedirs(SCREAMING_SNAKE_CASE       )
    A_      :				int		     =					default_yaml_config_file

  if config_file.endswith('''.json'''       ):
    config.to_json_file(SCREAMING_SNAKE_CASE       )
  else:
    config.to_yaml_file(SCREAMING_SNAKE_CASE       )
  print(f'''accelerate configuration saved at {config_file}'''       )


def   _SCREAMING_SNAKE_CASE	( ):
  A_      :				List[str]		     =					config_command_parser()
  A_      :				Any		     =					parser.parse_args()
  config_command(SCREAMING_SNAKE_CASE       )


if __name__ == "__main__":
     main()








import collections
import gzip
import os
import urllib

import numpy
from tensorflow.python.framework import dtypes, random_seed
from tensorflow.python.platform import gfile
from tensorflow.python.util.deprecation import deprecated

UpperCamelCase								=	collections.namedtuple("""_Datasets""", ["""train""", """validation""", """test"""])

# CVDF mirror of http://yann.lecun.com/exdb/mnist/
UpperCamelCase								=	"""https://storage.googleapis.com/cvdf-datasets/mnist/"""


def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE       ):
  A_      :				Any		     =					numpy.dtype(numpy.uintaa       ).newbyteorder('''>'''       )
  return numpy.frombuffer(bytestream.read(4       ) ,			dtype=SCREAMING_SNAKE_CASE       )[0]


@deprecated(SCREAMING_SNAKE_CASE ,			'''Please use tf.data to implement this functionality.'''       )
def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE       ):
  print('''Extracting''' ,			f.name       )
  with gzip.GzipFile(fileobj=SCREAMING_SNAKE_CASE       ) as bytestream:
    A_      :				Union[str, Any]		     =					_readaa(SCREAMING_SNAKE_CASE       )
    if magic != 2_051:
      raise ValueError(
          '''Invalid magic number %d in MNIST image file: %s''' % (magic, f.name)       )
    A_      :				str		     =					_readaa(SCREAMING_SNAKE_CASE       )
    A_      :				Tuple		     =					_readaa(SCREAMING_SNAKE_CASE       )
    A_      :				Dict		     =					_readaa(SCREAMING_SNAKE_CASE       )
    A_      :				Tuple		     =					bytestream.read(rows * cols * num_images       )
    A_      :				Dict		     =					numpy.frombuffer(SCREAMING_SNAKE_CASE ,			dtype=numpy.uinta       )
    A_      :				Union[str, Any]		     =					data.reshape(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			1       )
    return data


@deprecated(SCREAMING_SNAKE_CASE ,			'''Please use tf.one_hot on tensors.'''       )
def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE       ):
  A_      :				Optional[Any]		     =					labels_dense.shape[0]
  A_      :				List[str]		     =					numpy.arange(SCREAMING_SNAKE_CASE       ) * num_classes
  A_      :				int		     =					numpy.zeros((num_labels, num_classes)       )
  A_      :				Tuple		     =					1
  return labels_one_hot


@deprecated(SCREAMING_SNAKE_CASE ,			'''Please use tf.data to implement this functionality.'''       )
def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE=False ,			SCREAMING_SNAKE_CASE=10       ):
  print('''Extracting''' ,			f.name       )
  with gzip.GzipFile(fileobj=SCREAMING_SNAKE_CASE       ) as bytestream:
    A_      :				Any		     =					_readaa(SCREAMING_SNAKE_CASE       )
    if magic != 2_049:
      raise ValueError(
          '''Invalid magic number %d in MNIST label file: %s''' % (magic, f.name)       )
    A_      :				Tuple		     =					_readaa(SCREAMING_SNAKE_CASE       )
    A_      :				List[Any]		     =					bytestream.read(SCREAMING_SNAKE_CASE       )
    A_      :				int		     =					numpy.frombuffer(SCREAMING_SNAKE_CASE ,			dtype=numpy.uinta       )
    if one_hot:
      return _dense_to_one_hot(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE       )
    return labels






class   _lowerCamelCase :




       """simple docstring"""



       @deprecated(
           _SCREAMING_SNAKE_CASE				,       '''Please use alternatives such as official/mnist/_DataSet.py'''
           ''' from tensorflow/models.'''				,       )
       def __init__(      self				,       _SCREAMING_SNAKE_CASE				,       _SCREAMING_SNAKE_CASE				,       _SCREAMING_SNAKE_CASE=False				,       _SCREAMING_SNAKE_CASE=False				,       _SCREAMING_SNAKE_CASE=dtypes.floataa				,       _SCREAMING_SNAKE_CASE=True				,       _SCREAMING_SNAKE_CASE=None				,       )->Tuple:




         '''simple docstring'''





         A_     ,       A_      :				List[str]		     =					random_seed.get_seed(_SCREAMING_SNAKE_CASE		)
         # If op level seed is not set, use whatever graph level seed is returned
         numpy.random.seed(seeda if seed is None else seeda		)
         A_      :				Tuple		     =					dtypes.as_dtype(_SCREAMING_SNAKE_CASE		).base_dtype
         if dtype not in (dtypes.uinta, dtypes.floataa):
           raise TypeError('''Invalid image dtype %r, expected uint8 or float32''' % dtype		)
         if fake_data:
           A_      :				Optional[Any]		     =					1_0000
           A_      :				List[str]		     =					one_hot
         else:
           assert (
               images.shape[0] == labels.shape[0]
           ), F'''images.shape: {images.shape} labels.shape: {labels.shape}'''
           A_      :				List[str]		     =					images.shape[0]

           # Convert shape from [num examples, rows, columns, depth]
           # to [num examples, rows*columns] (assuming depth == 1)
           if reshape:
             assert images.shape[3] == 1
             A_      :				int		     =					images.reshape(
                 images.shape[0]				,       images.shape[1] * images.shape[2]		)
           if dtype == dtypes.floataa:
             # Convert from [0, 255] -> [0.0, 1.0].
             A_      :				Optional[int]		     =					images.astype(numpy.floataa		)
             A_      :				List[str]		     =					numpy.multiply(_SCREAMING_SNAKE_CASE				,       1.0 / 2_5_5.0		)
         A_      :				int		     =					images
         A_      :				Optional[int]		     =					labels
         A_      :				List[str]		     =					0
         A_      :				List[Any]		     =					0



       @property
       def        _snake_case       (      self		)->Optional[int]:




         '''simple docstring'''





         return self._images



       @property
       def        _snake_case       (      self		)->Optional[Any]:




         '''simple docstring'''





         return self._labels



       @property
       def        _snake_case       (      self		)->Union[str, Any]:




         '''simple docstring'''





         return self._num_examples



       @property
       def        _snake_case       (      self		)->Union[str, Any]:




         '''simple docstring'''





         return self._epochs_completed







       def        _snake_case       (      self				,       _SCREAMING_SNAKE_CASE				,       _SCREAMING_SNAKE_CASE=False				,       _SCREAMING_SNAKE_CASE=True		)->str:




         '''simple docstring'''





         if fake_data:
           A_      :				Any		     =					[1] * 784
           A_      :				int		     =					[1] + [0] * 9 if self.one_hot else 0
           return (
               [fake_image for _ in range(_SCREAMING_SNAKE_CASE		)],
               [fake_label for _ in range(_SCREAMING_SNAKE_CASE		)],
           )
         A_      :				Any		     =					self._index_in_epoch
         # Shuffle for the first epoch
         if self._epochs_completed == 0 and start == 0 and shuffle:
           A_      :				Any		     =					numpy.arange(self._num_examples		)
           numpy.random.shuffle(_SCREAMING_SNAKE_CASE		)
           A_      :				Optional[Any]		     =					self.images[perma]
           A_      :				List[Any]		     =					self.labels[perma]
         # Go to the next epoch
         if start + batch_size > self._num_examples:
           # Finished epoch
           self._epochs_completed += 1
           # Get the rest examples in this epoch
           A_      :				Tuple		     =					self._num_examples - start
           A_      :				Union[str, Any]		     =					self._images[start : self._num_examples]
           A_      :				List[Any]		     =					self._labels[start : self._num_examples]
           # Shuffle the data
           if shuffle:
             A_      :				List[Any]		     =					numpy.arange(self._num_examples		)
             numpy.random.shuffle(_SCREAMING_SNAKE_CASE		)
             A_      :				Tuple		     =					self.images[perm]
             A_      :				int		     =					self.labels[perm]
           # Start next epoch
           A_      :				Tuple		     =					0
           A_      :				Optional[Any]		     =					batch_size - rest_num_examples
           A_      :				Tuple		     =					self._index_in_epoch
           A_      :				Union[str, Any]		     =					self._images[start:end]
           A_      :				Any		     =					self._labels[start:end]
           return (
               numpy.concatenate((images_rest_part, images_new_part)				,       axis=0		),
               numpy.concatenate((labels_rest_part, labels_new_part)				,       axis=0		),
           )
         else:
           self._index_in_epoch += batch_size
           A_      :				List[str]		     =					self._index_in_epoch
           return self._images[start:end], self._labels[start:end]




@deprecated(SCREAMING_SNAKE_CASE ,			'''Please write your own downloading logic.'''       )
def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE       ):
  if not gfile.Exists(SCREAMING_SNAKE_CASE       ):
    gfile.MakeDirs(SCREAMING_SNAKE_CASE       )
  A_      :				str		     =					os.path.join(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE       )
  if not gfile.Exists(SCREAMING_SNAKE_CASE       ):
    urllib.request.urlretrieve(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE       )  # noqa: S310
    with gfile.GFile(SCREAMING_SNAKE_CASE       ) as f:
      A_      :				Dict		     =					f.size()
    print('''Successfully downloaded''' ,			SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			'''bytes.'''       )
  return filepath







@deprecated(
    SCREAMING_SNAKE_CASE ,			'''Please use alternatives such as:''' ''' tensorflow_datasets.load(\'mnist\')'''       )
def   _SCREAMING_SNAKE_CASE	( SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE=False ,			SCREAMING_SNAKE_CASE=False ,			SCREAMING_SNAKE_CASE=dtypes.floataa ,			SCREAMING_SNAKE_CASE=True ,			SCREAMING_SNAKE_CASE=5_000 ,			SCREAMING_SNAKE_CASE=None ,			SCREAMING_SNAKE_CASE=DEFAULT_SOURCE_URL ,			):
  if fake_data:

    def fake():
      return _DataSet(
          [] ,			[] ,			fake_data=SCREAMING_SNAKE_CASE ,			one_hot=SCREAMING_SNAKE_CASE ,			dtype=SCREAMING_SNAKE_CASE ,			seed=SCREAMING_SNAKE_CASE       )

    A_      :				List[str]		     =					fake()
    A_      :				Tuple		     =					fake()
    A_      :				Union[str, Any]		     =					fake()
    return _Datasets(train=SCREAMING_SNAKE_CASE ,			validation=SCREAMING_SNAKE_CASE ,			test=SCREAMING_SNAKE_CASE       )

  if not source_url:  # empty string check
    A_      :				List[str]		     =					DEFAULT_SOURCE_URL

  A_      :				List[Any]		     =					'''train-images-idx3-ubyte.gz'''
  A_      :				Tuple		     =					'''train-labels-idx1-ubyte.gz'''
  A_      :				Optional[int]		     =					'''t10k-images-idx3-ubyte.gz'''
  A_      :				Any		     =					'''t10k-labels-idx1-ubyte.gz'''

  A_      :				Dict		     =					_maybe_download(
      SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			source_url + train_images_file       )
  with gfile.Open(SCREAMING_SNAKE_CASE ,			'''rb'''       ) as f:
    A_      :				Optional[int]		     =					_extract_images(SCREAMING_SNAKE_CASE       )

  A_      :				List[str]		     =					_maybe_download(
      SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			source_url + train_labels_file       )
  with gfile.Open(SCREAMING_SNAKE_CASE ,			'''rb'''       ) as f:
    A_      :				Tuple		     =					_extract_labels(SCREAMING_SNAKE_CASE ,			one_hot=SCREAMING_SNAKE_CASE       )

  A_      :				Dict		     =					_maybe_download(
      SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			source_url + test_images_file       )
  with gfile.Open(SCREAMING_SNAKE_CASE ,			'''rb'''       ) as f:
    A_      :				List[str]		     =					_extract_images(SCREAMING_SNAKE_CASE       )

  A_      :				int		     =					_maybe_download(
      SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			source_url + test_labels_file       )
  with gfile.Open(SCREAMING_SNAKE_CASE ,			'''rb'''       ) as f:
    A_      :				Any		     =					_extract_labels(SCREAMING_SNAKE_CASE ,			one_hot=SCREAMING_SNAKE_CASE       )

  if not 0 <= validation_size <= len(SCREAMING_SNAKE_CASE       ):
    A_      :				str		     =					(
        '''Validation size should be between 0 and '''
        f'''{len(SCREAMING_SNAKE_CASE       )}. Received: {validation_size}.'''
    )
    raise ValueError(SCREAMING_SNAKE_CASE       )

  A_      :				Optional[Any]		     =					train_images[:validation_size]
  A_      :				Optional[Any]		     =					train_labels[:validation_size]
  A_      :				Any		     =					train_images[validation_size:]
  A_      :				Any		     =					train_labels[validation_size:]

  A_      :				Optional[Any]		     =					{'''dtype''': dtype, '''reshape''': reshape, '''seed''': seed}

  A_      :				List[str]		     =					_DataSet(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			**SCREAMING_SNAKE_CASE       )
  A_      :				Dict		     =					_DataSet(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			**SCREAMING_SNAKE_CASE       )
  A_      :				Dict		     =					_DataSet(SCREAMING_SNAKE_CASE ,			SCREAMING_SNAKE_CASE ,			**SCREAMING_SNAKE_CASE       )

  return _Datasets(train=SCREAMING_SNAKE_CASE ,			validation=SCREAMING_SNAKE_CASE ,			test=SCREAMING_SNAKE_CASE       )




