



import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


UpperCAmelCase							=       500_000

UpperCAmelCase			,							UpperCAmelCase							=       os.path.split(__file__)
UpperCAmelCase							=       os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))

@get_duration
def       A_   (						__a  :					datasets.Dataset		):

				"""simple docstring"""
				a__					   =		dataset.select(range(0			,		len(__a		)			,		2		)		)

@get_duration
def       A_   (						__a  :					datasets.Dataset		):

				"""simple docstring"""
				a__					   =		dataset.sort("""numbers"""		)

@get_duration
def       A_   (						__a  :					datasets.Dataset		):

				"""simple docstring"""
				a__					   =		dataset.shuffle()

@get_duration
def       A_   (						__a  :					datasets.Dataset		):

				"""simple docstring"""
				a__					   =		dataset.train_test_split(0.1		)

@get_duration
def       A_   (						__a  :					datasets.Dataset			,		__a  :					Any=10		):

				"""simple docstring"""
				for shard_id in range(__a		):
								a__					   =		dataset.shard(__a			,		__a		)

def       A_   (						):

				"""simple docstring"""
				a__					   =		{"""num examples""": SPEED_TEST_N_EXAMPLES}
				a__					   =		(select, sort, shuffle, train_test_split, shard)
				with tempfile.TemporaryDirectory() as tmp_dir:
								print("""generating dataset"""		)
								a__					   =		datasets.Features({"""text""": datasets.Value("""string"""		), """numbers""": datasets.Value("""float32"""		)}		)
								a__					   =		generate_example_dataset(
								    os.path.join(__a			,		"""dataset.arrow"""		)			,		__a			,		num_examples=__a		)
								print("""Functions"""		)
								for func in functions:
												print(func.__name__		)
												a__					   =		func(__a		)

				with open(__a			,		"""wb"""		) as f:
								f.write(json.dumps(__a		).encode("""utf-8"""		)		)


if __name__ == "__main__":  # useful to run the profiler
	benchmark_indices_mapping()








import json
import os
import shutil
import tempfile
import unittest

from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor, WavaVecaProcessor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_wavaveca import floats_list



class  SCREAMING_SNAKE_CASE_  ( unittest.TestCase			):


  """simple docstring"""





  def 			__A      (  self						)		->   Optional[int]:





       '''simple docstring'''







       __magic_name__				=       '''<pad> <s> </s> <unk> | E T A O N I H S R D L U M W C F G Y P B V K \' X J Q Z'''.split(''' '''						)
       __magic_name__				=       dict(zip(A	,  range(len(A						)						)						)						)

       __magic_name__				=       {
           '''pad_token''': '''<pad>''',
           '''unk_token''': '''<unk>''',
           '''bos_token''': '''<s>''',
           '''eos_token''': '''</s>''',
       }
       __magic_name__				=       {
           '''feature_size''': 1,
           '''padding_value''': 0.0,
           '''sampling_rate''': 1_60_00,
           '''return_attention_mask''': False,
           '''do_normalize''': True,
       }

       __magic_name__				=       tempfile.mkdtemp()
       __magic_name__				=       os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES['''vocab_file''']						)
       __magic_name__				=       os.path.join(self.tmpdirname	,  A						)
       with open(self.vocab_file	,  '''w'''	,  encoding='''utf-8'''						) as fp:
            fp.write(json.dumps(A						) + '''\n'''						)

       with open(self.feature_extraction_file	,  '''w'''	,  encoding='''utf-8'''						) as fp:
            fp.write(json.dumps(A						) + '''\n'''						)





  def 			__A      (  self	,  **A						)		->   str:





       '''simple docstring'''







       __magic_name__				=       self.add_kwargs_tokens_map.copy()
       kwargs.update(A						)
       return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname	,  **A						)





  def 			__A      (  self	,  **A						)		->   List[Any]:





       '''simple docstring'''







       return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname	,  **A						)





  def 			__A      (  self						)		->   List[Any]:





       '''simple docstring'''







       shutil.rmtree(self.tmpdirname						)





  def 			__A      (  self						)		->   List[Any]:





       '''simple docstring'''







       __magic_name__				=       self.get_tokenizer()
       __magic_name__				=       self.get_feature_extractor()

       __magic_name__				=       WavaVecaProcessor(tokenizer=A	,  feature_extractor=A						)

       processor.save_pretrained(self.tmpdirname						)
       __magic_name__				=       WavaVecaProcessor.from_pretrained(self.tmpdirname						)

       self.assertEqual(processor.tokenizer.get_vocab()	,  tokenizer.get_vocab()						)
       self.assertIsInstance(processor.tokenizer	,  A						)

       self.assertEqual(processor.feature_extractor.to_json_string()	,  feature_extractor.to_json_string()						)
       self.assertIsInstance(processor.feature_extractor	,  A						)





  def 			__A      (  self						)		->   Dict:





       '''simple docstring'''







       __magic_name__				=       WavaVecaProcessor(tokenizer=self.get_tokenizer()	,  feature_extractor=self.get_feature_extractor()						)
       processor.save_pretrained(self.tmpdirname						)

       __magic_name__				=       self.get_tokenizer(bos_token='''(BOS)'''	,  eos_token='''(EOS)'''						)
       __magic_name__				=       self.get_feature_extractor(do_normalize=A	,  padding_value=1.0						)

       __magic_name__				=       WavaVecaProcessor.from_pretrained(
           self.tmpdirname	,  bos_token='''(BOS)'''	,  eos_token='''(EOS)'''	,  do_normalize=A	,  padding_value=1.0						)

       self.assertEqual(processor.tokenizer.get_vocab()	,  tokenizer_add_kwargs.get_vocab()						)
       self.assertIsInstance(processor.tokenizer	,  A						)

       self.assertEqual(processor.feature_extractor.to_json_string()	,  feature_extractor_add_kwargs.to_json_string()						)
       self.assertIsInstance(processor.feature_extractor	,  A						)





  def 			__A      (  self						)		->   Dict:





       '''simple docstring'''







       __magic_name__				=       self.get_feature_extractor()
       __magic_name__				=       self.get_tokenizer()

       __magic_name__				=       WavaVecaProcessor(tokenizer=A	,  feature_extractor=A						)

       __magic_name__				=       floats_list((3, 10_00)						)

       __magic_name__				=       feature_extractor(A	,  return_tensors='''np'''						)
       __magic_name__				=       processor(A	,  return_tensors='''np'''						)

       for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum()	,  input_processor[key].sum()	,  delta=1E-2						)





  def 			__A      (  self						)		->   Optional[Any]:





       '''simple docstring'''







       __magic_name__				=       self.get_feature_extractor()
       __magic_name__				=       self.get_tokenizer()

       __magic_name__				=       WavaVecaProcessor(tokenizer=A	,  feature_extractor=A						)

       __magic_name__				=       '''This is a test string'''

       __magic_name__				=       processor(text=A						)

       __magic_name__				=       tokenizer(A						)

       for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key]	,  encoded_processor[key]						)





  def 			__A      (  self						)		->   Tuple:





       '''simple docstring'''







       __magic_name__				=       self.get_feature_extractor()
       __magic_name__				=       self.get_tokenizer()

       __magic_name__				=       WavaVecaProcessor(tokenizer=A	,  feature_extractor=A						)

       __magic_name__				=       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       __magic_name__				=       processor.batch_decode(A						)
       __magic_name__				=       tokenizer.batch_decode(A						)

       self.assertListEqual(A	,  A						)







  def 			__A      (  self						)		->   Dict:





       '''simple docstring'''







       __magic_name__				=       self.get_feature_extractor()
       __magic_name__				=       self.get_tokenizer()

       __magic_name__				=       WavaVecaProcessor(tokenizer=A	,  feature_extractor=A						)

       self.assertListEqual(
           processor.model_input_names	,  feature_extractor.model_input_names	,  msg='''`processor` and `feature_extractor` model input names do not match'''	,  )