


'''simple docstring'''




import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode





def 							a_       (						lowercase							,						**lowercase    )			->Union[str, Any]:






    '''simple docstring'''

    return RunBeamCommand(
        args.dataset							,						args.name							,						args.cache_dir							,						args.beam_pipeline_options							,						args.data_dir							,						args.all_configs							,						args.save_info or args.save_infos							,						args.ignore_verifications							,						args.force_redownload							,						**__lowerCAmelCase							,						)





class _snake_case   ( A__   ):
       """simple docstring"""





       @staticmethod
       def 		__SCREAMING_SNAKE_CASE(   __A	)     ->Optional[Any]:



           """simple docstring"""







           __lowerCamelCase		:      str											=					parser.add_parser("run_beam"			,				help="Run a Beam dataset processing pipeline"	)
           run_beam_parser.add_argument("dataset"			,				type=__A			,				help="Name of the dataset to download"	)
           run_beam_parser.add_argument("--name"			,				type=__A			,				default=__A			,				help="Dataset config name"	)
           run_beam_parser.add_argument(
               "--cache_dir"			,				type=__A			,				default=__A			,				help="Cache directory where the datasets are stored"			,				)
           run_beam_parser.add_argument(
               "--beam_pipeline_options"			,				type=__A			,				default=""			,				help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`"			,				)
           run_beam_parser.add_argument(
               "--data_dir"			,				type=__A			,				default=__A			,				help="Can be used to specify a manual directory to get the files from"			,				)
           run_beam_parser.add_argument("--all_configs"			,				action="store_true"			,				help="Test all dataset configurations"	)
           run_beam_parser.add_argument("--save_info"			,				action="store_true"			,				help="Save the dataset infos file"	)
           run_beam_parser.add_argument(
               "--ignore_verifications"			,				action="store_true"			,				help="Run the test without checksums and splits checks"	)
           run_beam_parser.add_argument("--force_redownload"			,				action="store_true"			,				help="Force dataset redownload"	)
           # aliases
           run_beam_parser.add_argument("--save_infos"			,				action="store_true"			,				help="alias for save_info"	)
           run_beam_parser.set_defaults(func=__A	)







       def __init__(   self			,				__A			,				__A			,				__A			,				__A			,				__A			,				__A			,				__A			,				__A			,				__A			,				**__A			,				)     ->List[str]:



           """simple docstring"""







           __lowerCamelCase		:      int											=					dataset
           __lowerCamelCase		:      Dict											=					name
           __lowerCamelCase		:      Optional[Any]											=					cache_dir
           __lowerCamelCase		:      int											=					beam_pipeline_options
           __lowerCamelCase		:      Any											=					data_dir
           __lowerCamelCase		:      Union[str, Any]											=					all_configs
           __lowerCamelCase		:      List[Any]											=					save_infos
           __lowerCamelCase		:      Optional[Any]											=					ignore_verifications
           __lowerCamelCase		:      List[str]											=					force_redownload
           __lowerCamelCase		:      Optional[Any]											=					config_kwargs







       def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[str]:



           """simple docstring"""







           import apache_beam as beam

           if self._name is not None and self._all_configs:
               print("Both parameters `name` and `all_configs` can't be used at once."	)
               exit(1	)
           __lowerCamelCase		:      Optional[int]											=					self._dataset, self._name
           __lowerCamelCase		:      Dict											=					dataset_module_factory(__A	)
           __lowerCamelCase		:      List[str]											=					import_main_class(dataset_module.module_path	)
           __lowerCamelCase		:      List[DatasetBuilder]											=					[]
           if self._beam_pipeline_options:
               __lowerCamelCase		:      Optional[int]											=					beam.options.pipeline_options.PipelineOptions(
                   flags=[f'--{opt.strip()}' for opt in self._beam_pipeline_options.split(","	) if opt]	)
           else:
               __lowerCamelCase		:      str											=					None
           if self._all_configs and len(builder_cls.BUILDER_CONFIGS	) > 0:
               for builder_config in builder_cls.BUILDER_CONFIGS:
                   builders.append(
                       builder_cls(
                           config_name=builder_config.name			,				data_dir=self._data_dir			,				hash=dataset_module.hash			,				beam_options=__A			,				cache_dir=self._cache_dir			,				base_path=dataset_module.builder_kwargs.get("base_path"	)			,				)	)
           else:
               builders.append(
                   builder_cls(
                       config_name=__A			,				data_dir=self._data_dir			,				beam_options=__A			,				cache_dir=self._cache_dir			,				base_path=dataset_module.builder_kwargs.get("base_path"	)			,				**self._config_kwargs			,				)	)

           for builder in builders:
               builder.download_and_prepare(
                   download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                   if not self._force_redownload
                   else DownloadMode.FORCE_REDOWNLOAD			,				download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH	)			,				verification_mode=VerificationMode.NO_CHECKS
                   if self._ignore_verifications
                   else VerificationMode.ALL_CHECKS			,				try_from_hf_gcs=__A			,				)
               if self._save_infos:
                   builder._save_infos()

           print("Apache beam run successful."	)

           # If save_infos=True, the dataset infos file is created next to the loaded module file.
           # Let's move it to the original directory of the dataset script, to allow the user to
           # upload them on S3 at the same time afterwards.
           if self._save_infos:
               __lowerCamelCase		:      str											=					os.path.join(builder_cls.get_imported_module_dir()			,				config.DATASETDICT_INFOS_FILENAME	)

               __lowerCamelCase		:      int											=					Path(__A	).name + ".py"

               __lowerCamelCase		:      int											=					os.path.join(__A			,				__A	)
               if os.path.isfile(__A	):
                   __lowerCamelCase		:      List[str]											=					os.path.dirname(__A	)
               elif os.path.isfile(__A	):
                   __lowerCamelCase		:      Optional[int]											=					path
               else:  # in case of a remote dataset
                   print(f'Dataset Infos file saved at {dataset_infos_path}'	)
                   exit(1	)

               # Move datasetinfo back to the user
               __lowerCamelCase		:      int											=					os.path.join(__A			,				config.DATASETDICT_INFOS_FILENAME	)
               copyfile(__A			,				__A	)
               print(f'Dataset Infos file saved at {user_dataset_infos_path}'	)









'''simple docstring'''




import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
				import torch


__lowercase  :      Any      		=				random.Random()





def 							a_       (						lowercase							,						lowercase=1.0							,						lowercase=None							,						lowercase=None    )			->int:






				'''simple docstring'''

				if rng is None:
								__lowerCamelCase		:      Any											=					global_rng

				__lowerCamelCase		:      List[Any]											=					[]
				for batch_idx in range(shape[0]    ):
								values.append([]    )
								for _ in range(shape[1]    ):
												values[-1].append(rng.random() * scale    )

				return values





@require_torch
class _snake_case   ( unittest.TestCase   ):
							"""simple docstring"""





							def __init__(   self			,				__A			,				__A=7			,				__A=400			,				__A=2000			,				__A=1			,				__A=0.0			,				__A=24000			,				__A=True			,				)     ->Optional[int]:



											"""simple docstring"""







											__lowerCamelCase		:      Dict											=					parent
											__lowerCamelCase		:      Tuple											=					batch_size
											__lowerCamelCase		:      str											=					min_seq_length
											__lowerCamelCase		:      Optional[Any]											=					max_seq_length
											__lowerCamelCase		:      int											=					(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
											__lowerCamelCase		:      List[Any]											=					feature_size
											__lowerCamelCase		:      int											=					padding_value
											__lowerCamelCase		:      Optional[Any]											=					sampling_rate
											__lowerCamelCase		:      List[str]											=					return_attention_mask







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Optional[Any]:



											"""simple docstring"""







											return {
											    "feature_size": self.feature_size,
											    "padding_value": self.padding_value,
											    "sampling_rate": self.sampling_rate,
											    "return_attention_mask": self.return_attention_mask,
											}







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A=False			,				__A=False	)     ->int:



											"""simple docstring"""







											def _flatten(__A	):
															return list(itertools.chain(*__A	)	)

											if equal_length:
															__lowerCamelCase		:      Optional[int]											=					floats_list((self.batch_size, self.max_seq_length)	)
											else:
															# make sure that inputs increase in size
															__lowerCamelCase		:      Optional[Any]											=					[
															    _flatten(floats_list((x, self.feature_size)	)	)
															    for x in range(self.min_seq_length			,				self.max_seq_length			,				self.seq_length_diff	)
															]

											if numpify:
															__lowerCamelCase		:      int											=					[np.asarray(__A	) for x in audio_inputs]

											return audio_inputs





@require_torch
class _snake_case   ( a__    ,   unittest.TestCase   ):
							"""simple docstring"""





							__a												=     EncodecFeatureExtractor







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					EnCodecFeatureExtractionTester(self	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[int]											=					self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
											# create three inputs of length 800, 1000, and 1200
											__lowerCamelCase		:      Optional[Any]											=					[floats_list((1, x)	)[0] for x in range(800			,				1400			,				200	)]
											__lowerCamelCase		:      Optional[Any]											=					[np.asarray(__A	) for audio_input in audio_inputs]

											# Test not batched input
											__lowerCamelCase		:      List[str]											=					feat_extract(audio_inputs[0]			,				return_tensors="np"	).input_values
											__lowerCamelCase		:      Union[str, Any]											=					feat_extract(np_audio_inputs[0]			,				return_tensors="np"	).input_values
											self.assertTrue(np.allclose(__A			,				__A			,				atol=1E-3	)	)

											# Test batched
											__lowerCamelCase		:      Optional[int]											=					feat_extract(__A			,				padding=__A			,				return_tensors="np"	).input_values
											__lowerCamelCase		:      List[str]											=					feat_extract(__A			,				padding=__A			,				return_tensors="np"	).input_values
											for enc_seq_a, enc_seq_a in zip(__A			,				__A	):
															self.assertTrue(np.allclose(__A			,				__A			,				atol=1E-3	)	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[Any]											=					self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
											__lowerCamelCase		:      Tuple											=					np.random.rand(100	).astype(np.floataa	)
											__lowerCamelCase		:      Any											=					np_audio_inputs.tolist()

											for inputs in [py_audio_inputs, np_audio_inputs]:
															__lowerCamelCase		:      int											=					feature_extractor.pad([{"input_values": inputs}]			,				return_tensors="np"	)
															self.assertTrue(np_processed.input_values.dtype == np.floataa	)
															__lowerCamelCase		:      List[Any]											=					feature_extractor.pad([{"input_values": inputs}]			,				return_tensors="pt"	)
															self.assertTrue(pt_processed.input_values.dtype == torch.floataa	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->int:



											"""simple docstring"""







											from datasets import load_dataset

											__lowerCamelCase		:      List[Any]											=					load_dataset("hf-internal-testing/librispeech_asr_dummy"			,				"clean"			,				split="validation"	)
											# automatic decoding with librispeech
											__lowerCamelCase		:      str											=					ds.sort("id"	).select(range(__A	)	)[:num_samples]["audio"]

											return [x["array"] for x in audio_samples]







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:      Dict											=					torch.tensor(
											    [2.3804E-03, 2.0752E-03, 1.9836E-03, 2.1057E-03, 1.6174E-03,
											     3.0518E-04, 9.1553E-05, 3.3569E-04, 9.7656E-04, 1.8311E-03,
											     2.0142E-03, 2.1057E-03, 1.7395E-03, 4.5776E-04, -3.9673E-04,
											     4.5776E-04, 1.0071E-03, 9.1553E-05, 4.8828E-04, 1.1597E-03,
											     7.3242E-04, 9.4604E-04, 1.8005E-03, 1.8311E-03, 8.8501E-04,
											     4.2725E-04, 4.8828E-04, 7.3242E-04, 1.0986E-03, 2.1057E-03]	)
											# fmt: on
											__lowerCamelCase		:      Union[str, Any]											=					self._load_datasamples(1	)
											__lowerCamelCase		:      str											=					EncodecFeatureExtractor()
											__lowerCamelCase		:      Optional[int]											=					feature_extractor(__A			,				return_tensors="pt"	).input_values
											self.assertEquals(input_values.shape			,				(1, 1, 93680)	)
											self.assertTrue(torch.allclose(input_values[0, 0, :30]			,				__A			,				atol=1E-6	)	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[Any]											=					torch.tensor(
											    [2.3804E-03, 2.0752E-03, 1.9836E-03, 2.1057E-03, 1.6174E-03,
											     3.0518E-04, 9.1553E-05, 3.3569E-04, 9.7656E-04, 1.8311E-03,
											     2.0142E-03, 2.1057E-03, 1.7395E-03, 4.5776E-04, -3.9673E-04,
											     4.5776E-04, 1.0071E-03, 9.1553E-05, 4.8828E-04, 1.1597E-03,
											     7.3242E-04, 9.4604E-04, 1.8005E-03, 1.8311E-03, 8.8501E-04,
											     4.2725E-04, 4.8828E-04, 7.3242E-04, 1.0986E-03, 2.1057E-03]	)
											# fmt: on
											__lowerCamelCase		:      Union[str, Any]											=					self._load_datasamples(1	)
											__lowerCamelCase		:      str											=					[np.tile(input_audio[0][None]			,				reps=(2, 1)	)]
											input_audio[0][1] *= 0.5
											__lowerCamelCase		:      Optional[int]											=					EncodecFeatureExtractor(feature_size=2	)
											__lowerCamelCase		:      str											=					feature_extractor(__A			,				return_tensors="pt"	).input_values
											self.assertEquals(input_values.shape			,				(1, 2, 93680)	)
											self.assertTrue(torch.allclose(input_values[0, 0, :30]			,				__A			,				atol=1E-6	)	)
											self.assertTrue(torch.allclose(input_values[0, 1, :30]			,				EXPECTED_INPUT_VALUES * 0.5			,				atol=1E-6	)	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					self._load_datasamples(2	)
											# would be easier if the stride was like
											__lowerCamelCase		:      List[Any]											=					EncodecFeatureExtractor(feature_size=1			,				chunk_length_s=1			,				overlap=0.01	)

											# pad and trunc raise an error ?
											with self.assertRaisesRegex(
											    __A			,				"^Both padding and truncation were set. Make sure you only set one.$"			,				):
															__lowerCamelCase		:      Dict											=					feature_extractor(
															    __A			,				padding="max_length"			,				truncation=__A			,				return_tensors="pt"	).input_values

											# truncate to chunk
											__lowerCamelCase		:      Optional[Any]											=					feature_extractor(__A			,				truncation=__A			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(2, 1, 71520)	)  # 2 chunks

											# force truncate to max_length
											__lowerCamelCase		:      List[str]											=					feature_extractor(
											    __A			,				truncation=__A			,				max_length=48000			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(2, 1, 48000)	)

											# pad to chunk
											__lowerCamelCase		:      List[str]											=					feature_extractor(__A			,				padding=__A			,				return_tensors="pt"	).input_values
											self.assertEquals(padded_outputs.shape			,				(2, 1, 95280)	)

											# pad to chunk
											__lowerCamelCase		:      Optional[Any]											=					feature_extractor(__A			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(2, 1, 95280)	)

											# force pad to max length
											__lowerCamelCase		:      int											=					feature_extractor(
											    __A			,				padding="max_length"			,				max_length=100000			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(2, 1, 100000)	)

											# force no pad
											with self.assertRaisesRegex(
											    __A			,				"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"			,				):
															__lowerCamelCase		:      Optional[Any]											=					feature_extractor(__A			,				padding=__A			,				return_tensors="pt"	).input_values

											__lowerCamelCase		:      int											=					feature_extractor(input_audio[0]			,				padding=__A			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(1, 1, 93680)	)

											# no pad if no chunk_length_s
											__lowerCamelCase		:      Any											=					None
											with self.assertRaisesRegex(
											    __A			,				"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"			,				):
															__lowerCamelCase		:      int											=					feature_extractor(__A			,				padding=__A			,				return_tensors="pt"	).input_values

											__lowerCamelCase		:      Any											=					feature_extractor(input_audio[0]			,				padding=__A			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(1, 1, 93680)	)

											# no pad if no overlap
											__lowerCamelCase		:      List[Any]											=					2
											__lowerCamelCase		:      Any											=					None
											with self.assertRaisesRegex(
											    __A			,				"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"			,				):
															__lowerCamelCase		:      Optional[int]											=					feature_extractor(__A			,				padding=__A			,				return_tensors="pt"	).input_values

											__lowerCamelCase		:      List[str]											=					feature_extractor(input_audio[0]			,				padding=__A			,				return_tensors="pt"	).input_values
											self.assertEquals(truncated_outputs.shape			,				(1, 1, 93680)	)





