





'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__    :      Tuple						=    get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_torch_available():
  from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowercase__    :      Tuple						=    25_00_04
lowercase__    :      Optional[Any]						=    25_00_20


@require_sentencepiece
@require_tokenizers
class  SCREAMING_SNAKE_CASE    (__snake_case					,		unittest.TestCase							):
     lowerCAmelCase					=       MBartaaTokenizer
     lowerCAmelCase					=       MBartaaTokenizerFast
     lowerCAmelCase					=       True
     lowerCAmelCase					=       True


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          super().setUp()

          # We have a SentencePiece fixture for testing
          __A						: Any      =       MBartaaTokenizer(__SCREAMING_SNAKE_CASE						,	src_lang='en_XX'						,	tgt_lang='ro_RO'						,	keep_accents=__SCREAMING_SNAKE_CASE)
          tokenizer.save_pretrained(self.tmpdirname)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[Any]      =       '<s>'
          __A						: Any      =       0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE)						,	__SCREAMING_SNAKE_CASE)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE)						,	__SCREAMING_SNAKE_CASE)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       list(self.get_tokenizer().get_vocab().keys())

          self.assertEqual(vocab_keys[0]						,	'<s>')
          self.assertEqual(vocab_keys[1]						,	'<pad>')
          self.assertEqual(vocab_keys[-1]						,	'<mask>')
          self.assertEqual(len(__SCREAMING_SNAKE_CASE)						,	1054)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertEqual(self.get_tokenizer().vocab_size						,	1054)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       MBartaaTokenizer(__SCREAMING_SNAKE_CASE						,	src_lang='en_XX'						,	tgt_lang='ro_RO'						,	keep_accents=__SCREAMING_SNAKE_CASE)

          __A						: Optional[Any]      =       tokenizer.tokenize('This is a test')
          self.assertListEqual(__SCREAMING_SNAKE_CASE						,	['▁This', '▁is', '▁a', '▁t', 'est'])

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE)						,	[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,	)

          __A						: str      =       tokenizer.tokenize('I was born in 92000, and this is falsé.')
          self.assertListEqual(
              __SCREAMING_SNAKE_CASE						,	[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']						,	)
          __A						: Dict      =       tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE)
          self.assertListEqual(
              __SCREAMING_SNAKE_CASE						,	[
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]						,	)

          __A						: Optional[int]      =       tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE)
          self.assertListEqual(
              __SCREAMING_SNAKE_CASE						,	[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']						,	)


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[Any]      =       {'input_ids': [[25_0004, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [25_0004, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_0004, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=__SCREAMING_SNAKE_CASE						,	model_name='facebook/mbart-large-50'						,	revision='d3913889c59cd5c9e456b269c376325eabad57e2'						,	)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if not self.test_slow_tokenizer:
               # as we don't have a slow version, we can't compare the outputs between slow and fast versions
               return

          __A						: List[str]      =       (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-mbart50', {})
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                    __A						: str      =       self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE						,	**__SCREAMING_SNAKE_CASE)
                    __A						: Tuple      =       self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE						,	**__SCREAMING_SNAKE_CASE)

                    __A						: Tuple      =       tempfile.mkdtemp()

                    __A						: Tuple      =       tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE)
                    __A						: List[Any]      =       tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE)

                    # Checks it save with the same files + the tokenizer.json file for the fast one
                    self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))
                    __A						: str      =       tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f)
                    self.assertSequenceEqual(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE)

                    # Checks everything loads correctly in the same way
                    __A						: Union[str, Any]      =       tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE)
                    __A						: Tuple      =       tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE)

                    # Check special tokens are set accordingly on Rust and Python
                    for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE))
                         # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                         # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                    shutil.rmtree(__SCREAMING_SNAKE_CASE)

                    # Save tokenizer rust, legacy_format=True
                    __A						: int      =       tempfile.mkdtemp()

                    __A						: Any      =       tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE						,	legacy_format=__SCREAMING_SNAKE_CASE)
                    __A						: List[Any]      =       tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE)

                    # Checks it save with the same files
                    self.assertSequenceEqual(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE)

                    # Checks everything loads correctly in the same way
                    __A						: Union[str, Any]      =       tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE)
                    __A						: Dict      =       tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE)

                    # Check special tokens are set accordingly on Rust and Python
                    for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE))

                    shutil.rmtree(__SCREAMING_SNAKE_CASE)

                    # Save tokenizer rust, legacy_format=False
                    __A						: Tuple      =       tempfile.mkdtemp()

                    __A						: Dict      =       tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE						,	legacy_format=__SCREAMING_SNAKE_CASE)
                    __A						: Dict      =       tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE)

                    # Checks it saved the tokenizer.json file
                    self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))

                    # Checks everything loads correctly in the same way
                    __A						: Tuple      =       tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE)
                    __A						: Tuple      =       tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE)

                    # Check special tokens are set accordingly on Rust and Python
                    for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE))

                    shutil.rmtree(__SCREAMING_SNAKE_CASE)




@require_torch
@require_sentencepiece
@require_tokenizers
class  SCREAMING_SNAKE_CASE    (unittest.TestCase							):
     lowerCAmelCase					=       """facebook/mbart-large-50-one-to-many-mmt"""
     lowerCAmelCase					=       [
         """ UN Chief Says There Is No Military Solution in Syria""",
         """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
     ]
     lowerCAmelCase					=       [
         """Şeful ONU declară că nu există o soluţie militară în Siria""",
         """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
         """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
         """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
     ]
     lowerCAmelCase					=       [EN_CODE, 8274, 12_7873, 2_5916, 7, 8622, 2071, 438, 6_7485, 53, 18_7895, 23, 5_1712, 2]


     @classmethod
     def 							SCREAMING_SNAKE_CASE  (						cls):

          '''simple docstring'''






          __A						: Tuple      =       MBartaaTokenizer.from_pretrained(
              cls.checkpoint_name						,	src_lang='en_XX'						,	tgt_lang='ro_RO')
          __A						: Optional[Any]      =       1
          return cls


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ar_AR']						,	25_0001)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['en_EN']						,	25_0004)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ro_RO']						,	25_0020)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['mr_IN']						,	25_0038)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[Any]      =       self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
          self.assertListEqual(self.expected_src_tokens						,	__SCREAMING_SNAKE_CASE)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertIn(__SCREAMING_SNAKE_CASE						,	self.tokenizer.all_special_ids)
          __A						: int      =       [RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
          __A						: Tuple      =       self.tokenizer.decode(__SCREAMING_SNAKE_CASE						,	skip_special_tokens=__SCREAMING_SNAKE_CASE)
          __A						: Union[str, Any]      =       self.tokenizer.decode(generated_ids[1:]						,	skip_special_tokens=__SCREAMING_SNAKE_CASE)
          self.assertEqual(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE)
          self.assertNotIn(self.tokenizer.eos_token						,	__SCREAMING_SNAKE_CASE)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[int]      =       ['this is gunna be a long sentence ' * 20]
          assert isinstance(src_text[0]						,	__SCREAMING_SNAKE_CASE)
          __A						: Any      =       10
          __A						: Dict      =       self.tokenizer(__SCREAMING_SNAKE_CASE						,	max_length=__SCREAMING_SNAKE_CASE						,	truncation=__SCREAMING_SNAKE_CASE).input_ids[0]
          self.assertEqual(ids[0]						,	__SCREAMING_SNAKE_CASE)
          self.assertEqual(ids[-1]						,	2)
          self.assertEqual(len(__SCREAMING_SNAKE_CASE)						,	__SCREAMING_SNAKE_CASE)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR'])						,	[25_0053, 25_0001])


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Union[str, Any]      =       tempfile.mkdtemp()
          __A						: List[Any]      =       self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
          __A						: Any      =       MBartaaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,	__SCREAMING_SNAKE_CASE)


     @require_torch
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Tuple      =       self.tokenizer(self.src_text						,	text_target=self.tgt_text						,	padding=__SCREAMING_SNAKE_CASE						,	return_tensors='pt')
          __A						: Tuple      =       shift_tokens_right(batch['labels']						,	self.tokenizer.pad_token_id)

          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          assert batch.input_ids[1][0] == EN_CODE
          assert batch.input_ids[1][-1] == 2
          assert batch.labels[1][0] == RO_CODE
          assert batch.labels[1][-1] == 2
          assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]


     @require_torch
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: str      =       self.tokenizer(
              self.src_text						,	text_target=self.tgt_text						,	padding=__SCREAMING_SNAKE_CASE						,	truncation=__SCREAMING_SNAKE_CASE						,	max_length=len(self.expected_src_tokens)						,	return_tensors='pt'						,	)
          __A						: Any      =       shift_tokens_right(batch['labels']						,	self.tokenizer.pad_token_id)

          self.assertIsInstance(__SCREAMING_SNAKE_CASE						,	__SCREAMING_SNAKE_CASE)

          self.assertEqual((2, 14)						,	batch.input_ids.shape)
          self.assertEqual((2, 14)						,	batch.attention_mask.shape)
          __A						: Optional[Any]      =       batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens						,	__SCREAMING_SNAKE_CASE)
          self.assertEqual(2						,	batch.decoder_input_ids[0, 0])  # decoder_start_token_id
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens						,	[EN_CODE])
          self.assertEqual(self.tokenizer.suffix_tokens						,	[self.tokenizer.eos_token_id])


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[int]      =       self.tokenizer(self.src_text						,	padding=__SCREAMING_SNAKE_CASE						,	truncation=__SCREAMING_SNAKE_CASE						,	max_length=3						,	return_tensors='pt')
          __A						: Optional[int]      =       self.tokenizer(
              text_target=self.tgt_text						,	padding=__SCREAMING_SNAKE_CASE						,	truncation=__SCREAMING_SNAKE_CASE						,	max_length=10						,	return_tensors='pt')
          __A						: Union[str, Any]      =       targets['input_ids']
          __A						: Optional[int]      =       shift_tokens_right(__SCREAMING_SNAKE_CASE						,	self.tokenizer.pad_token_id)

          self.assertEqual(batch.input_ids.shape[1]						,	3)
          self.assertEqual(batch.decoder_input_ids.shape[1]						,	10)


     @require_torch
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: str      =       self.tokenizer._build_translation_inputs(
              'A test'						,	return_tensors='pt'						,	src_lang='en_XX'						,	tgt_lang='ar_AR')

          self.assertEqual(
              nested_simplify(__SCREAMING_SNAKE_CASE)						,	{
                  # en_XX, A, test, EOS
                  'input_ids': [[25_0004, 62, 3034, 2]],
                  'attention_mask': [[1, 1, 1, 1]],
                  # ar_AR
                  'forced_bos_token_id': 25_0001,
              }						,	)







"""simple docstring"""





import itertools
import random
import unittest

import numpy as np

from transformers import BatchFeature, SpeechTaFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
  import torch


_SCREAMING_SNAKE_CASE					:				Tuple					    =						random.Random()







def      lowerCamelCase__       (							_lowerCamelCase       :		List[Any]		,				_lowerCamelCase       :		List[Any]=1.0		,				_lowerCamelCase       :		Optional[int]=None		,				_lowerCamelCase       :		str=None )      ->				List[str]:
  if rng is None:
    lowerCamelCase_         =						global_rng

  lowerCamelCase_         =						[]
  for batch_idx in range(shape[0] ):
    values.append([] )
    for _ in range(shape[1] ):
      values[-1].append(rng.random() * scale )

  return values






@require_torch
class      a						(					unittest.TestCase		):






    def __init__(       self     :							Any       ,						__SCREAMING_SNAKE_CASE     :							Dict       ,						__SCREAMING_SNAKE_CASE     :							Tuple=7       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]=400       ,						__SCREAMING_SNAKE_CASE     :							int=2000       ,						__SCREAMING_SNAKE_CASE     :							List[Any]=1       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]=0.0       ,						__SCREAMING_SNAKE_CASE     :							Dict=16000       ,						__SCREAMING_SNAKE_CASE     :							Dict=True       ,						__SCREAMING_SNAKE_CASE     :							List[Any]=80       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]=16       ,						__SCREAMING_SNAKE_CASE     :							int=64       ,						__SCREAMING_SNAKE_CASE     :							Tuple="hann_window"       ,						__SCREAMING_SNAKE_CASE     :							Dict=80       ,						__SCREAMING_SNAKE_CASE     :							List[str]=7600       ,						__SCREAMING_SNAKE_CASE     :							List[str]=1e-1_0       ,						__SCREAMING_SNAKE_CASE     :							Any=True       ,						)     ->		List[str]:
      lowerCamelCase_         =						parent
      lowerCamelCase_         =						batch_size
      lowerCamelCase_         =						min_seq_length
      lowerCamelCase_         =						max_seq_length
      lowerCamelCase_         =						(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
      lowerCamelCase_         =						feature_size
      lowerCamelCase_         =						padding_value
      lowerCamelCase_         =						sampling_rate
      lowerCamelCase_         =						do_normalize
      lowerCamelCase_         =						num_mel_bins
      lowerCamelCase_         =						hop_length
      lowerCamelCase_         =						win_length
      lowerCamelCase_         =						win_function
      lowerCamelCase_         =						fmin
      lowerCamelCase_         =						fmax
      lowerCamelCase_         =						mel_floor
      lowerCamelCase_         =						return_attention_mask






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		List[Any]:
      return {
          "feature_size": self.feature_size,
          "padding_value": self.padding_value,
          "sampling_rate": self.sampling_rate,
          "do_normalize": self.do_normalize,
          "num_mel_bins": self.num_mel_bins,
          "hop_length": self.hop_length,
          "win_length": self.win_length,
          "win_function": self.win_function,
          "fmin": self.fmin,
          "fmax": self.fmax,
          "mel_floor": self.mel_floor,
          "return_attention_mask": self.return_attention_mask,
      }






    def    UpperCamelCase       (       self     :							List[str]       ,						__SCREAMING_SNAKE_CASE     :							Tuple=False       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]=False					)     ->		str:
      def _flatten(__SCREAMING_SNAKE_CASE     :							Any					):
        return list(itertools.chain(*__SCREAMING_SNAKE_CASE					)					)

      if equal_length:
        lowerCamelCase_         =						floats_list((self.batch_size, self.max_seq_length)					)
      else:
        # make sure that inputs increase in size
        lowerCamelCase_         =						[
            _flatten(floats_list((x, self.feature_size)					)					)
            for x in range(self.min_seq_length       ,						self.max_seq_length       ,						self.seq_length_diff					)
        ]

      if numpify:
        lowerCamelCase_         =						[np.asarray(__SCREAMING_SNAKE_CASE					) for x in speech_inputs]

      return speech_inputs






    def    UpperCamelCase       (       self     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							str=False       ,						__SCREAMING_SNAKE_CASE     :							Any=False					)     ->		int:
      if equal_length:
        lowerCamelCase_         =						[floats_list((self.max_seq_length, self.num_mel_bins)					) for _ in range(self.batch_size					)]
      else:
        # make sure that inputs increase in size
        lowerCamelCase_         =						[
            floats_list((x, self.num_mel_bins)					)
            for x in range(self.min_seq_length       ,						self.max_seq_length       ,						self.seq_length_diff					)
        ]

      if numpify:
        lowerCamelCase_         =						[np.asarray(__SCREAMING_SNAKE_CASE					) for x in speech_inputs]

      return speech_inputs






@require_torch
class      a						(					__snake_case  ,      unittest.TestCase		):
    SCREAMING_SNAKE_CASE						:     Any						= SpeechTaFeatureExtractor






    def    UpperCamelCase       (       self     :							Any					)     ->		Any:
      lowerCamelCase_         =						SpeechTaFeatureExtractionTester(self					)






    def    UpperCamelCase       (       self     :							Dict       ,						__SCREAMING_SNAKE_CASE     :							int					)     ->		List[Any]:
      self.assertTrue(np.all(np.mean(__SCREAMING_SNAKE_CASE       ,						axis=0					) < 1e-3					)					)
      self.assertTrue(np.all(np.abs(np.var(__SCREAMING_SNAKE_CASE       ,						axis=0					) - 1					) < 1e-3					)					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		int:
      # Tests that all call wrap to encode_plus and batch_encode_plus
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      # create three inputs of length 800, 1000, and 1200
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in range(800       ,						1400       ,						200					)]
      lowerCamelCase_         =						[np.asarray(__SCREAMING_SNAKE_CASE					) for speech_input in speech_inputs]

      # Test not batched input
      lowerCamelCase_         =						feat_extract(speech_inputs[0]       ,						return_tensors='np'					).input_values
      lowerCamelCase_         =						feat_extract(np_speech_inputs[0]       ,						return_tensors='np'					).input_values
      self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-3					)					)

      # Test batched
      lowerCamelCase_         =						feat_extract(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      lowerCamelCase_         =						feat_extract(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-3					)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		Optional[Any]:
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in range(800       ,						1400       ,						200					)]

      lowerCamelCase_         =						['longest', 'max_length', 'do_not_pad']
      lowerCamelCase_         =						[None, 1600, None]
      for max_length, padding in zip(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						feat_extract(__SCREAMING_SNAKE_CASE       ,						padding=__SCREAMING_SNAKE_CASE       ,						max_length=__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					)
        lowerCamelCase_         =						processed.input_values

        self._check_zero_mean_unit_variance(input_values[0][:800]					)
        self.assertTrue(input_values[0][800:].sum() < 1e-6					)
        self._check_zero_mean_unit_variance(input_values[1][:1000]					)
        self.assertTrue(input_values[0][1000:].sum() < 1e-6					)
        self._check_zero_mean_unit_variance(input_values[2][:1200]					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Dict:
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      lowerCamelCase_         =						range(800       ,						1400       ,						200					)
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in lengths]

      lowerCamelCase_         =						['longest', 'max_length', 'do_not_pad']
      lowerCamelCase_         =						[None, 1600, None]

      for max_length, padding in zip(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						feat_extract(__SCREAMING_SNAKE_CASE       ,						max_length=__SCREAMING_SNAKE_CASE       ,						padding=__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						processed.input_values

        self._check_zero_mean_unit_variance(input_values[0][:800]					)
        self._check_zero_mean_unit_variance(input_values[1][:1000]					)
        self._check_zero_mean_unit_variance(input_values[2][:1200]					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Optional[int]:
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in range(800       ,						1400       ,						200					)]
      lowerCamelCase_         =						feat_extract(
          __SCREAMING_SNAKE_CASE       ,						truncation=__SCREAMING_SNAKE_CASE       ,						max_length=1000       ,						padding='max_length'       ,						return_tensors='np'					)
      lowerCamelCase_         =						processed.input_values

      self._check_zero_mean_unit_variance(input_values[0, :800]					)
      self._check_zero_mean_unit_variance(input_values[1]					)
      self._check_zero_mean_unit_variance(input_values[2]					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in range(800       ,						1400       ,						200					)]
      lowerCamelCase_         =						feat_extract(
          __SCREAMING_SNAKE_CASE       ,						truncation=__SCREAMING_SNAKE_CASE       ,						max_length=1000       ,						padding='longest'       ,						return_tensors='np'					)
      lowerCamelCase_         =						processed.input_values

      self._check_zero_mean_unit_variance(input_values[0, :800]					)
      self._check_zero_mean_unit_variance(input_values[1, :1000]					)
      self._check_zero_mean_unit_variance(input_values[2]					)

      # make sure that if max_length < longest -> then pad to max_length
      self.assertTrue(input_values.shape == (3, 1000)					)

      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in range(800       ,						1400       ,						200					)]
      lowerCamelCase_         =						feat_extract(
          __SCREAMING_SNAKE_CASE       ,						truncation=__SCREAMING_SNAKE_CASE       ,						max_length=2000       ,						padding='longest'       ,						return_tensors='np'					)
      lowerCamelCase_         =						processed.input_values

      self._check_zero_mean_unit_variance(input_values[0, :800]					)
      self._check_zero_mean_unit_variance(input_values[1, :1000]					)
      self._check_zero_mean_unit_variance(input_values[2]					)

      # make sure that if max_length > longest -> then pad to longest
      self.assertTrue(input_values.shape == (3, 1200)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		List[Any]:
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      lowerCamelCase_         =						np.random.rand(100					).astype(np.floataa					)
      lowerCamelCase_         =						np_speech_inputs.tolist()

      for inputs in [py_speech_inputs, np_speech_inputs]:
        lowerCamelCase_         =						feature_extractor.pad([{'input_values': inputs}]       ,						return_tensors='np'					)
        self.assertTrue(np_processed.input_values.dtype == np.floataa					)
        lowerCamelCase_         =						feature_extractor.pad([{'input_values': inputs}]       ,						return_tensors='pt'					)
        self.assertTrue(pt_processed.input_values.dtype == torch.floataa					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Tuple:
      # Tests that all call wrap to encode_plus and batch_encode_plus
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
      # create three inputs of length 800, 1000, and 1200
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in range(800       ,						1400       ,						200					)]
      lowerCamelCase_         =						[np.asarray(__SCREAMING_SNAKE_CASE					) for speech_input in speech_inputs]

      # Test feature size
      lowerCamelCase_         =						feature_extractor(audio_target=__SCREAMING_SNAKE_CASE       ,						padding=__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      self.assertTrue(input_values.ndim == 3					)
      self.assertTrue(input_values.shape[-1] == feature_extractor.num_mel_bins					)

      # Test not batched input
      lowerCamelCase_         =						feature_extractor(speech_inputs[0]       ,						return_tensors='np'					).input_values
      lowerCamelCase_         =						feature_extractor(np_speech_inputs[0]       ,						return_tensors='np'					).input_values
      self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-3					)					)

      # Test batched
      lowerCamelCase_         =						feature_extractor(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      lowerCamelCase_         =						feature_extractor(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-3					)					)

      # Test 2-D numpy arrays are batched.
      lowerCamelCase_         =						[floats_list((1, x)					)[0] for x in (800, 800, 800)]
      lowerCamelCase_         =						np.asarray(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						feature_extractor(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      lowerCamelCase_         =						feature_extractor(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					).input_values
      for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-3					)					)






    def    UpperCamelCase       (       self     :							int					)     ->		Union[str, Any]:
      lowerCamelCase_         =						self.feat_extract_tester.prepare_inputs_for_target()
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_dict					)
      lowerCamelCase_         =						feat_extract.model_input_names[0]

      lowerCamelCase_         =						BatchFeature({input_name: speech_inputs}					)

      self.assertTrue(all(len(__SCREAMING_SNAKE_CASE					) == len(__SCREAMING_SNAKE_CASE					) for x, y in zip(__SCREAMING_SNAKE_CASE       ,						processed_features[input_name]					)					)					)

      lowerCamelCase_         =						self.feat_extract_tester.prepare_inputs_for_target(equal_length=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						BatchFeature({input_name: speech_inputs}       ,						tensor_type='np'					)

      lowerCamelCase_         =						processed_features[input_name]

      if len(batch_features_input.shape					) < 3:
        lowerCamelCase_         =						batch_features_input[:, :, None]

      self.assertTrue(
          batch_features_input.shape
          == (self.feat_extract_tester.batch_size, len(speech_inputs[0]					), self.feat_extract_tester.num_mel_bins)					)






    @require_torch
    def    UpperCamelCase       (       self     :							Tuple					)     ->		Any:
      lowerCamelCase_         =						self.feat_extract_tester.prepare_inputs_for_target(equal_length=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_dict					)
      lowerCamelCase_         =						feat_extract.model_input_names[0]

      lowerCamelCase_         =						BatchFeature({input_name: speech_inputs}       ,						tensor_type='pt'					)

      lowerCamelCase_         =						processed_features[input_name]

      if len(batch_features_input.shape					) < 3:
        lowerCamelCase_         =						batch_features_input[:, :, None]

      self.assertTrue(
          batch_features_input.shape
          == (self.feat_extract_tester.batch_size, len(speech_inputs[0]					), self.feat_extract_tester.num_mel_bins)					)






    @require_torch
    def    UpperCamelCase       (       self     :							List[Any]					)     ->		List[Any]:
      lowerCamelCase_         =						self.feature_extraction_class(**self.feat_extract_dict					)
      lowerCamelCase_         =						self.feat_extract_tester.prepare_inputs_for_target()
      lowerCamelCase_         =						feat_extract.model_input_names[0]

      lowerCamelCase_         =						BatchFeature({input_name: speech_inputs}					)

      lowerCamelCase_         =						feat_extract.num_mel_bins  # hack!

      lowerCamelCase_         =						feat_extract.pad(__SCREAMING_SNAKE_CASE       ,						padding='longest'       ,						return_tensors='np'					)[input_name]
      lowerCamelCase_         =						feat_extract.pad(__SCREAMING_SNAKE_CASE       ,						padding='longest'       ,						return_tensors='pt'					)[input_name]

      self.assertTrue(abs(input_np.astype(np.floataa					).sum() - input_pt.numpy().astype(np.floataa					).sum()					) < 1e-2					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Union[str, Any]:
      lowerCamelCase_         =						self.feat_extract_dict
      lowerCamelCase_         =						True
      lowerCamelCase_         =						self.feature_extraction_class(**__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.feat_extract_tester.prepare_inputs_for_target()
      lowerCamelCase_         =						[len(__SCREAMING_SNAKE_CASE					) for x in speech_inputs]
      lowerCamelCase_         =						feat_extract.model_input_names[0]

      lowerCamelCase_         =						BatchFeature({input_name: speech_inputs}					)

      lowerCamelCase_         =						feat_extract.num_mel_bins  # hack!

      lowerCamelCase_         =						feat_extract.pad(__SCREAMING_SNAKE_CASE       ,						padding='longest'       ,						return_tensors='np'					)
      self.assertIn('attention_mask'       ,						__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(list(processed.attention_mask.shape					)       ,						list(processed[input_name].shape[:2]					)					)
      self.assertListEqual(processed.attention_mask.sum(-1					).tolist()       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Optional[Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						self.feat_extract_dict
      lowerCamelCase_         =						True
      lowerCamelCase_         =						self.feature_extraction_class(**__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.feat_extract_tester.prepare_inputs_for_target()
      lowerCamelCase_         =						[len(__SCREAMING_SNAKE_CASE					) for x in speech_inputs]
      lowerCamelCase_         =						feat_extract.model_input_names[0]

      lowerCamelCase_         =						BatchFeature({input_name: speech_inputs}					)
      lowerCamelCase_         =						min(__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						feat_extract.num_mel_bins  # hack!

      lowerCamelCase_         =						feat_extract.pad(
          __SCREAMING_SNAKE_CASE       ,						padding='max_length'       ,						max_length=__SCREAMING_SNAKE_CASE       ,						truncation=__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					)
      self.assertIn('attention_mask'       ,						__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(
          list(processed_pad.attention_mask.shape					)       ,						[processed_pad[input_name].shape[0], max_length]					)
      self.assertListEqual(
          processed_pad.attention_mask[:, :max_length].sum(-1					).tolist()       ,						[max_length for x in speech_inputs]					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]					)     ->		List[str]:
      from datasets import load_dataset

      lowerCamelCase_         =						load_dataset('hf-internal-testing/librispeech_asr_dummy'       ,						'clean'       ,						split='validation'					)
      # automatic decoding with librispeech
      lowerCamelCase_         =						ds.sort('id'					).select(range(__SCREAMING_SNAKE_CASE					)					)[:num_samples]['audio']

      return [x["array"] for x in speech_samples]






    def    UpperCamelCase       (       self     :							str					)     ->		Dict:
      # fmt: off
      lowerCamelCase_         =						torch.tensor(
          [2.3_8_0_4e-0_3, 2.0_7_5_2e-0_3, 1.9_8_3_6e-0_3, 2.1_0_5_7e-0_3, 1.6_1_7_4e-0_3,
           3.0_5_1_8e-0_4, 9.1_5_5_3e-0_5, 3.3_5_6_9e-0_4, 9.7_6_5_6e-0_4, 1.8_3_1_1e-0_3,
           2.0_1_4_2e-0_3, 2.1_0_5_7e-0_3, 1.7_3_9_5e-0_3, 4.5_7_7_6e-0_4, -3.9_6_7_3e-0_4,
           4.5_7_7_6e-0_4, 1.0_0_7_1e-0_3, 9.1_5_5_3e-0_5, 4.8_8_2_8e-0_4, 1.1_5_9_7e-0_3,
           7.3_2_4_2e-0_4, 9.4_6_0_4e-0_4, 1.8_0_0_5e-0_3, 1.8_3_1_1e-0_3, 8.8_5_0_1e-0_4,
           4.2_7_2_5e-0_4, 4.8_8_2_8e-0_4, 7.3_2_4_2e-0_4, 1.0_9_8_6e-0_3, 2.1_0_5_7e-0_3]					)
      # fmt: on

      lowerCamelCase_         =						self._load_datasamples(1					)
      lowerCamelCase_         =						SpeechTaFeatureExtractor()
      lowerCamelCase_         =						feature_extractor(__SCREAMING_SNAKE_CASE       ,						return_tensors='pt'					).input_values
      self.assertEquals(input_values.shape       ,						(1, 93680)					)
      self.assertTrue(torch.allclose(input_values[0, :30]       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-6					)					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[int]:
      # fmt: off
      lowerCamelCase_         =						torch.tensor(
          [-2.6_870, -3.0_104, -3.1_356, -3.5_352, -3.0_044, -3.0_353, -3.4_719, -3.6_777,
           -3.1_520, -2.9_435, -2.6_553, -2.8_795, -2.9_944, -2.5_921, -3.0_279, -3.0_386,
           -3.0_864, -3.1_291, -3.2_353, -2.7_444, -2.6_831, -2.7_287, -3.1_761, -3.1_571,
           -3.2_726, -3.0_582, -3.1_007, -3.4_533, -3.4_695, -3.0_998]					)
      # fmt: on

      lowerCamelCase_         =						self._load_datasamples(1					)
      lowerCamelCase_         =						SpeechTaFeatureExtractor()
      lowerCamelCase_         =						feature_extractor(audio_target=__SCREAMING_SNAKE_CASE       ,						return_tensors='pt'					).input_values
      self.assertEquals(input_values.shape       ,						(1, 366, 80)					)
      self.assertTrue(torch.allclose(input_values[0, 0, :30]       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-4					)					)




