"""simple docstring"""







import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
						import tensorflow as tf

						tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
						import torch

if is_datasets_available():
						from datasets import load_dataset


UpperCamelCase_       :       List[Any]							   =		5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors


def   a__      (		UpperCAmelCase__       :			Namespace			)->	Dict:
						return PTtoTFCommand(
						    args.model_name		,			args.local_dir		,			args.max_error		,			args.new_weights		,			args.no_pr		,			args.push		,			args.extra_commit_description		,			args.override_model_class		,			)







class 			_lowercase		(   __lowerCamelCase  ):

				@staticmethod
				def        A (     _lowercase       :    ArgumentParser     ):


										'''simple docstring'''
										UpperCAmelCase__						=		parser.add_parser(
										    """pt-to-tf"""       ,    help=(
										        """CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."""
										        """ Can also be used to validate existing weights without opening PRs, with --no-pr."""
										    )       ,    )
										train_parser.add_argument(
										    """--model-name"""       ,    type=_lowercase       ,    required=_lowercase       ,    help="""The model name, including owner/organization, as seen on the hub."""       ,    )
										train_parser.add_argument(
										    """--local-dir"""       ,    type=_lowercase       ,    default=""""""       ,    help="""Optional local directory of the model repository. Defaults to /tmp/{model_name}"""       ,    )
										train_parser.add_argument(
										    """--max-error"""       ,    type=_lowercase       ,    default=_lowercase       ,    help=(
										        f'Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'
										    )       ,    )
										train_parser.add_argument(
										    """--new-weights"""       ,    action="""store_true"""       ,    help="""Optional flag to create new TensorFlow weights, even if they already exist."""       ,    )
										train_parser.add_argument(
										    """--no-pr"""       ,    action="""store_true"""       ,    help="""Optional flag to NOT open a PR with converted weights."""     )
										train_parser.add_argument(
										    """--push"""       ,    action="""store_true"""       ,    help="""Optional flag to push the weights directly to `main` (requires permissions)"""       ,    )
										train_parser.add_argument(
										    """--extra-commit-description"""       ,    type=_lowercase       ,    default=""""""       ,    help="""Optional additional commit description to use when opening a PR (e.g. to tag the owner)."""       ,    )
										train_parser.add_argument(
										    """--override-model-class"""       ,    type=_lowercase       ,    default=_lowercase       ,    help="""If you think you know better than the auto-detector, you can specify the model class here. """
										    """Can be either an AutoModel class or a specific model class like BertForSequenceClassification."""       ,    )
										train_parser.set_defaults(func=_lowercase     )

				@staticmethod
				def        A (     _lowercase       :    str       ,    _lowercase       :    Dict     ):


										'''simple docstring'''
										UpperCAmelCase__						=		set(pt_outputs.keys()     )
										UpperCAmelCase__						=		set(tf_outputs.keys()     )
										if pt_out_attrs != tf_out_attrs:
																raise ValueError(
																    f'The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'
																    f' {tf_out_attrs})'     )

										# 2. For each output attribute, computes the difference
										def _find_pt_tf_differences(_lowercase       :    List[Any]       ,    _lowercase       :    List[Any]       ,    _lowercase       :    Any       ,    _lowercase       :    Dict=""     ):
																# If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
																# recursivelly, keeping the name of the attribute.
																if isinstance(_lowercase       ,    torch.Tensor     ):
																						UpperCAmelCase__						=		np.max(np.abs(pt_out.numpy() - tf_out.numpy()     )     )
																						UpperCAmelCase__						=		tensor_difference
																else:
																						UpperCAmelCase__						=		attr_name
																						for i, pt_item in enumerate(_lowercase     ):
																												# If it is a named attribute, we keep the name. Otherwise, just its index.
																												if isinstance(_lowercase       ,    _lowercase     ):
																																		UpperCAmelCase__						=		root_name + pt_item
																																		UpperCAmelCase__						=		tf_out[pt_item]
																																		UpperCAmelCase__						=		pt_out[pt_item]
																												else:
																																		UpperCAmelCase__						=		root_name + f'[{i}]'
																																		UpperCAmelCase__						=		tf_out[i]
																												UpperCAmelCase__						=		_find_pt_tf_differences(_lowercase       ,    _lowercase       ,    _lowercase       ,    _lowercase     )

																return differences

										return _find_pt_tf_differences(_lowercase       ,    _lowercase       ,    {}     )

				def __init__(     self       :    Optional[Any]       ,    _lowercase       :    str       ,    _lowercase       :    str       ,    _lowercase       :    float       ,    _lowercase       :    bool       ,    _lowercase       :    bool       ,    _lowercase       :    bool       ,    _lowercase       :    str       ,    _lowercase       :    str       ,    *_lowercase       :    Union[str, Any]       ,    ):


										'''simple docstring'''
										UpperCAmelCase__						=		logging.get_logger("""transformers-cli/pt_to_tf"""     )
										UpperCAmelCase__						=		model_name
										UpperCAmelCase__						=		local_dir if local_dir else os.path.join("""/tmp"""       ,    _lowercase     )
										UpperCAmelCase__						=		max_error
										UpperCAmelCase__						=		new_weights
										UpperCAmelCase__						=		no_pr
										UpperCAmelCase__						=		push
										UpperCAmelCase__						=		extra_commit_description
										UpperCAmelCase__						=		override_model_class

				def        A (     self       :    Tuple       ,    _lowercase       :    Optional[int]       ,    _lowercase       :    Tuple       ,    _lowercase       :    Optional[Any]     ):


										'''simple docstring'''

										def _get_audio_input():
																UpperCAmelCase__						=		load_dataset("""hf-internal-testing/librispeech_asr_dummy"""       ,    """clean"""       ,    split="""validation"""     )
																UpperCAmelCase__						=		ds.sort("""id"""     ).select(range(2     )     )[:2]["""audio"""]
																UpperCAmelCase__						=		[x["""array"""] for x in speech_samples]
																return raw_samples

										UpperCAmelCase__						=		type(pt_model.config     )
										if model_config_class in PROCESSOR_MAPPING:
																UpperCAmelCase__						=		AutoProcessor.from_pretrained(self._local_dir     )
																if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
																						UpperCAmelCase__						=		processor.tokenizer.eos_token
										elif model_config_class in IMAGE_PROCESSOR_MAPPING:
																UpperCAmelCase__						=		AutoImageProcessor.from_pretrained(self._local_dir     )
										elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
																UpperCAmelCase__						=		AutoFeatureExtractor.from_pretrained(self._local_dir     )
										elif model_config_class in TOKENIZER_MAPPING:
																UpperCAmelCase__						=		AutoTokenizer.from_pretrained(self._local_dir     )
																if processor.pad_token is None:
																						UpperCAmelCase__						=		processor.eos_token
										else:
																raise ValueError(f'Unknown data processing type (model config type: {model_config_class})'     )

										UpperCAmelCase__						=		set(inspect.signature(pt_model.forward     ).parameters.keys()     )
										UpperCAmelCase__						=		{}
										if "input_ids" in model_forward_signature:
																processor_inputs.update(
																    {
																        """text""": ["""Hi there!""", """I am a batch with more than one row and different input lengths."""],
																        """padding""": True,
																        """truncation""": True,
																    }     )
										if "pixel_values" in model_forward_signature:
																UpperCAmelCase__						=		load_dataset("""cifar10"""       ,    """plain_text"""       ,    split="""test"""     )[:2]["""img"""]
																processor_inputs.update({"""images""": sample_images}     )
										if "input_features" in model_forward_signature:
																UpperCAmelCase__						=		inspect.signature(processor.feature_extractor     ).parameters
																# Pad to the largest input length by default but take feature extractor default
																# padding value if it exists e.g. "max_length" and is not False or None
																if "padding" in feature_extractor_signature:
																						UpperCAmelCase__						=		feature_extractor_signature["""padding"""].default
																						if default_strategy is not False and default_strategy is not None:
																												UpperCAmelCase__						=		default_strategy
																						else:
																												UpperCAmelCase__						=		True
																else:
																						UpperCAmelCase__						=		True
																processor_inputs.update({"""audio""": _get_audio_input(), """padding""": padding_strategy}     )
										if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
																processor_inputs.update({"""audio""": _get_audio_input(), """padding""": True}     )
										UpperCAmelCase__						=		processor(**_lowercase       ,    return_tensors="""pt"""     )
										UpperCAmelCase__						=		processor(**_lowercase       ,    return_tensors="""tf"""     )

										# Extra input requirements, in addition to the input modality
										if (
										    config.is_encoder_decoder
										    or (hasattr(_lowercase       ,    """encoder"""     ) and hasattr(_lowercase       ,    """decoder"""     ))
										    or "decoder_input_ids" in tf_dummy_inputs
										):
																UpperCAmelCase__						=		np.asarray([[1], [1]]       ,    dtype=_lowercase     ) * (pt_model.config.decoder_start_token_id or 0)
																pt_input.update({"""decoder_input_ids""": torch.tensor(_lowercase     )}     )
																tf_input.update({"""decoder_input_ids""": tf.convert_to_tensor(_lowercase     )}     )

										return pt_input, tf_input

				def        A (     self       :    Any     ):


										'''simple docstring'''
										# hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
										if version.parse(huggingface_hub.__version__     ) < version.parse("""0.9.0"""     ):
																raise ImportError(
																    """The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"""
																    """ installation."""     )
										else:
																from huggingface_hub import Repository, create_commit
																from huggingface_hub._commit_api import CommitOperationAdd

										# Fetch remote data
										UpperCAmelCase__						=		Repository(local_dir=self._local_dir       ,    clone_from=self._model_name     )

										# Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
										UpperCAmelCase__						=		AutoConfig.from_pretrained(self._local_dir     )
										UpperCAmelCase__						=		config.architectures
										if self._override_model_class is not None:
																if self._override_model_class.startswith("""TF"""     ):
																						UpperCAmelCase__						=		[self._override_model_class[2:]]
																else:
																						UpperCAmelCase__						=		[self._override_model_class]
																try:
																						UpperCAmelCase__						=		getattr(import_module("""transformers"""     )       ,    architectures[0]     )
																except AttributeError:
																						raise ValueError(f'Model class {self._override_model_class} not found in transformers.'     )
																try:
																						UpperCAmelCase__						=		getattr(import_module("""transformers"""     )       ,    """TF""" + architectures[0]     )
																except AttributeError:
																						raise ValueError(f'TF model class TF{self._override_model_class} not found in transformers.'     )
										elif architectures is None:  # No architecture defined -- use auto classes
																UpperCAmelCase__						=		getattr(import_module("""transformers"""     )       ,    """AutoModel"""     )
																UpperCAmelCase__						=		getattr(import_module("""transformers"""     )       ,    """TFAutoModel"""     )
																self._logger.warning("""No detected architecture, using AutoModel/TFAutoModel"""     )
										else:  # Architecture defined -- use it
																if len(_lowercase     ) > 1:
																						raise ValueError(f'More than one architecture was found, aborting. (architectures = {architectures})'     )
																self._logger.warning(f'Detected architecture: {architectures[0]}'     )
																UpperCAmelCase__						=		getattr(import_module("""transformers"""     )       ,    architectures[0]     )
																try:
																						UpperCAmelCase__						=		getattr(import_module("""transformers"""     )       ,    """TF""" + architectures[0]     )
																except AttributeError:
																						raise AttributeError(f'The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.'     )

        # Check the TF dummy inputs to see what keys we need in the forward pass
										UpperCAmelCase__						=		tf_class.from_config(_lowercase     )
										UpperCAmelCase__						=		tf_from_pt_model.dummy_inputs

										del tf_from_pt_model  # Try to keep only one model in memory at a time

										# Load the model and get some basic inputs
										UpperCAmelCase__						=		pt_class.from_pretrained(self._local_dir     )
										pt_model.eval()

										UpperCAmelCase__	,      UpperCAmelCase__						=		self.get_inputs(_lowercase       ,    _lowercase       ,    _lowercase     )

										with torch.no_grad():
																UpperCAmelCase__						=		pt_model(**_lowercase       ,    output_hidden_states=_lowercase     )
										del pt_model  # will no longer be used, and may have a large memory footprint

										UpperCAmelCase__						=		tf_class.from_pretrained(self._local_dir       ,    from_pt=_lowercase     )
										UpperCAmelCase__						=		tf_from_pt_model(**_lowercase       ,    output_hidden_states=_lowercase       ,    training=_lowercase     )

										# Confirms that cross loading PT weights into TF worked.
										UpperCAmelCase__						=		self.find_pt_tf_differences(_lowercase       ,    _lowercase     )
										UpperCAmelCase__						=		{k: v for k, v in crossload_differences.items() if """hidden""" not in k}
										UpperCAmelCase__						=		{k: v for k, v in crossload_differences.items() if """hidden""" in k}
										if len(_lowercase     ) == 0 and architectures is not None:
																raise ValueError(
																    f'Something went wrong -- the config file has architectures ({architectures}), but no model head'
																    """ output was found. All outputs start with 'hidden'"""     )
										UpperCAmelCase__						=		max(output_differences.values()     ) if output_differences else 0.0
										UpperCAmelCase__						=		max(hidden_differences.values()     )
										if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
																raise ValueError(
																    """The cross-loaded TensorFlow model has different outputs, something went wrong!\n"""
																    + f'\nList of maximum output differences above the threshold ({self._max_error}):\n'
																    + """\n""".join([f'{k}: {v:.3e}' for k, v in output_differences.items() if v > self._max_error]     )
																    + f'\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'
																    + """\n""".join([f'{k}: {v:.3e}' for k, v in hidden_differences.items() if v > self._max_error]     )     )

										# Save the weights in a TF format (if needed) and confirms that the results are still good
										UpperCAmelCase__						=		os.path.join(self._local_dir       ,    _lowercase     )
										UpperCAmelCase__						=		os.path.join(self._local_dir       ,    _lowercase     )
										if (not os.path.exists(_lowercase     ) and not os.path.exists(_lowercase     )) or self._new_weights:
																tf_from_pt_model.save_pretrained(self._local_dir     )
										del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

										UpperCAmelCase__						=		tf_class.from_pretrained(self._local_dir     )
										UpperCAmelCase__						=		tf_model(**_lowercase       ,    output_hidden_states=_lowercase     )

										UpperCAmelCase__						=		self.find_pt_tf_differences(_lowercase       ,    _lowercase     )
										UpperCAmelCase__						=		{k: v for k, v in conversion_differences.items() if """hidden""" not in k}
										UpperCAmelCase__						=		{k: v for k, v in conversion_differences.items() if """hidden""" in k}
										if len(_lowercase     ) == 0 and architectures is not None:
																raise ValueError(
																    f'Something went wrong -- the config file has architectures ({architectures}), but no model head'
																    """ output was found. All outputs start with 'hidden'"""     )
										UpperCAmelCase__						=		max(output_differences.values()     ) if output_differences else 0.0
										UpperCAmelCase__						=		max(hidden_differences.values()     )
										if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
																raise ValueError(
																    """The converted TensorFlow model has different outputs, something went wrong!\n"""
																    + f'\nList of maximum output differences above the threshold ({self._max_error}):\n'
																    + """\n""".join([f'{k}: {v:.3e}' for k, v in output_differences.items() if v > self._max_error]     )
																    + f'\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'
																    + """\n""".join([f'{k}: {v:.3e}' for k, v in hidden_differences.items() if v > self._max_error]     )     )

										UpperCAmelCase__						=		"""Update TF weights""" if self._new_weights else """Add TF weights"""
										if self._push:
																repo.git_add(auto_lfs_track=_lowercase     )
																repo.git_commit(_lowercase     )
																repo.git_push(blocking=_lowercase     )  # this prints a progress bar with the upload
																self._logger.warning(f'TF weights pushed into {self._model_name}'     )
										elif not self._no_pr:
																self._logger.warning("""Uploading the weights into a new PR..."""     )
																UpperCAmelCase__						=		(
																    """Model converted by the [`transformers`' `pt_to_tf`"""
																    """ CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). """
																    """All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"""
																    f'Maximum crossload output difference={max_crossload_output_diff:.3e}; '
																    f'Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'
																    f'Maximum conversion output difference={max_conversion_output_diff:.3e}; '
																    f'Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'
																)
																if self._max_error > MAX_ERROR:
																						commit_descrition += (
																						    f'\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'
																						)
																if self._extra_commit_description:
																						commit_descrition += "\n\n" + self._extra_commit_description

																# sharded model -> adds all related files (index and .h5 shards)
																if os.path.exists(_lowercase     ):
																						UpperCAmelCase__						=		[
																						    CommitOperationAdd(path_in_repo=_lowercase       ,    path_or_fileobj=_lowercase     )
																						]
																						for shard_path in tf.io.gfile.glob(self._local_dir + """/tf_model-*.h5"""     ):
																												operations += [
																												    CommitOperationAdd(path_in_repo=os.path.basename(_lowercase     )       ,    path_or_fileobj=_lowercase     )
																												]
																else:
																						UpperCAmelCase__						=		[CommitOperationAdd(path_in_repo=_lowercase       ,    path_or_fileobj=_lowercase     )]

																UpperCAmelCase__						=		create_commit(
																    repo_id=self._model_name       ,    operations=_lowercase       ,    commit_message=_lowercase       ,    commit_description=_lowercase       ,    repo_type="""model"""       ,    create_pr=_lowercase       ,    ).pr_url
																self._logger.warning(f'PR open in {hub_pr_url}'     )


"""simple docstring"""







import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile, rmtree
from typing import Generator

import datasets.config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.filelock import logger as fl_logger
from datasets.utils.info_utils import VerificationMode
from datasets.utils.logging import ERROR, get_logger


UpperCamelCase_       :       Tuple							   =		get_logger(__name__)


def   a__      (		UpperCAmelCase__       :			Optional[int]			)->	Dict:
						return TestCommand(
						    args.dataset		,			args.name		,			args.cache_dir		,			args.data_dir		,			args.all_configs		,			args.save_info or args.save_infos		,			args.ignore_verifications		,			args.force_redownload		,			args.clear_cache		,			)







class 			_lowercase		(   __lowerCamelCase  ):
				lowerCamelCase__						=       False  # to tell pytest it's not a test class

				@staticmethod
				def        A (     _lowercase       :    ArgumentParser     ):


										'''simple docstring'''
										UpperCAmelCase__						=		parser.add_parser("""test"""       ,    help="""Test dataset implementation."""     )
										test_parser.add_argument("""--name"""       ,    type=_lowercase       ,    default=_lowercase       ,    help="""Dataset processing name"""     )
										test_parser.add_argument(
										    """--cache_dir"""       ,    type=_lowercase       ,    default=_lowercase       ,    help="""Cache directory where the datasets are stored."""       ,    )
										test_parser.add_argument(
										    """--data_dir"""       ,    type=_lowercase       ,    default=_lowercase       ,    help="""Can be used to specify a manual directory to get the files from."""       ,    )
										test_parser.add_argument("""--all_configs"""       ,    action="""store_true"""       ,    help="""Test all dataset configurations"""     )
										test_parser.add_argument(
										    """--save_info"""       ,    action="""store_true"""       ,    help="""Save the dataset infos in the dataset card (README.md)"""     )
										test_parser.add_argument(
										    """--ignore_verifications"""       ,    action="""store_true"""       ,    help="""Run the test without checksums and splits checks."""       ,    )
										test_parser.add_argument("""--force_redownload"""       ,    action="""store_true"""       ,    help="""Force dataset redownload"""     )
										test_parser.add_argument(
										    """--clear_cache"""       ,    action="""store_true"""       ,    help="""Remove downloaded files and cached datasets after each config test"""       ,    )
										# aliases
										test_parser.add_argument("""--save_infos"""       ,    action="""store_true"""       ,    help="""alias to save_info"""     )
										test_parser.add_argument("""dataset"""       ,    type=_lowercase       ,    help="""Name of the dataset to download"""     )
										test_parser.set_defaults(func=_test_command_factory     )

				def __init__(     self       :    List[Any]       ,    _lowercase       :    str       ,    _lowercase       :    str       ,    _lowercase       :    str       ,    _lowercase       :    str       ,    _lowercase       :    bool       ,    _lowercase       :    bool       ,    _lowercase       :    bool       ,    _lowercase       :    bool       ,    _lowercase       :    bool       ,    ):


										'''simple docstring'''
										UpperCAmelCase__						=		dataset
										UpperCAmelCase__						=		name
										UpperCAmelCase__						=		cache_dir
										UpperCAmelCase__						=		data_dir
										UpperCAmelCase__						=		all_configs
										UpperCAmelCase__						=		save_infos
										UpperCAmelCase__						=		ignore_verifications
										UpperCAmelCase__						=		force_redownload
										UpperCAmelCase__						=		clear_cache
										if clear_cache and not cache_dir:
																print(
																    """When --clear_cache is used, specifying a cache directory is mandatory.\n"""
																    """The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n"""
																    """Please provide a --cache_dir that will be used to test the dataset script."""     )
																exit(1     )
										if save_infos:
																UpperCAmelCase__						=		True

				def        A (     self       :    Optional[int]     ):


										'''simple docstring'''
										fl_logger().setLevel(_lowercase     )
										if self._name is not None and self._all_configs:
																print("""Both parameters `config` and `all_configs` can't be used at once."""     )
																exit(1     )
										UpperCAmelCase__	,      UpperCAmelCase__						=		self._dataset, self._name
										UpperCAmelCase__						=		dataset_module_factory(_lowercase     )
										UpperCAmelCase__						=		import_main_class(module.module_path     )
										UpperCAmelCase__						=		len(builder_cls.BUILDER_CONFIGS     ) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1

										def get_builders() -> Generator[DatasetBuilder, None, None]:
																if self._all_configs and builder_cls.BUILDER_CONFIGS:
																						for i, config in enumerate(builder_cls.BUILDER_CONFIGS     ):
																												if "config_name" in module.builder_kwargs:
																																		yield builder_cls(
																																		    cache_dir=self._cache_dir       ,    data_dir=self._data_dir       ,    **module.builder_kwargs       ,    )
																												else:
																																		yield builder_cls(
																																		    config_name=config.name       ,    cache_dir=self._cache_dir       ,    data_dir=self._data_dir       ,    **module.builder_kwargs       ,    )
																else:
																						if "config_name" in module.builder_kwargs:
																												yield builder_cls(cache_dir=self._cache_dir       ,    data_dir=self._data_dir       ,    **module.builder_kwargs     )
																						else:
																												yield builder_cls(
																												    config_name=_lowercase       ,    cache_dir=self._cache_dir       ,    data_dir=self._data_dir       ,    **module.builder_kwargs       ,    )

										for j, builder in enumerate(get_builders()     ):
																print(f'Testing builder \'{builder.config.name}\' ({j + 1}/{n_builders})'     )
																UpperCAmelCase__						=		os.path.exists(
																    os.path.join(builder.get_imported_module_dir()       ,    datasets.config.DATASETDICT_INFOS_FILENAME     )     )  # record checksums only if we need to update a (deprecated) dataset_infos.json
																builder.download_and_prepare(
																    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
																    if not self._force_redownload
																    else DownloadMode.FORCE_REDOWNLOAD       ,    verification_mode=VerificationMode.NO_CHECKS
																    if self._ignore_verifications
																    else VerificationMode.ALL_CHECKS       ,    try_from_hf_gcs=_lowercase       ,    )
																builder.as_dataset()
																if self._save_infos:
																						builder._save_infos()

																# If save_infos=True, the dataset card (README.md) is created next to the loaded module file.
																# The dataset_infos are saved in the YAML part of the README.md

																# Let's move it to the original directory of the dataset script, to allow the user to
																# upload them on S3 at the same time afterwards.
																if self._save_infos:
																						UpperCAmelCase__						=		os.path.join(builder_cls.get_imported_module_dir()       ,    """README.md"""     )
																						UpperCAmelCase__						=		Path(_lowercase     ).name + """.py"""
																						UpperCAmelCase__						=		os.path.join(_lowercase       ,    _lowercase     )
																						if os.path.isfile(_lowercase     ):
																												UpperCAmelCase__						=		os.path.dirname(_lowercase     )
																						elif os.path.isfile(_lowercase     ):
																												UpperCAmelCase__						=		path
																						elif os.path.isdir(_lowercase     ):  # for local directories containing only data files
																												UpperCAmelCase__						=		path
																						else:  # in case of a remote dataset
																												UpperCAmelCase__						=		None
																												print(f'Dataset card saved at {dataset_readme_path}'     )

																						# Move dataset_info back to the user
																						if dataset_dir is not None:
																												UpperCAmelCase__						=		os.path.join(_lowercase       ,    """README.md"""     )
																												copyfile(_lowercase       ,    _lowercase     )
																												print(f'Dataset card saved at {user_dataset_readme_path}'     )

            # If clear_cache=True, the download folder and the dataset builder cache directory are deleted
																if self._clear_cache:
																						if os.path.isdir(builder._cache_dir     ):
																												logger.warning(f'Clearing cache at {builder._cache_dir}'     )
																												rmtree(builder._cache_dir     )
																						UpperCAmelCase__						=		os.path.join(self._cache_dir       ,    datasets.config.DOWNLOADED_DATASETS_DIR     )
																						if os.path.isdir(_lowercase     ):
																												logger.warning(f'Clearing cache at {download_dir}'     )
																												rmtree(_lowercase     )

										print("""Test successful."""     )

