





import itertools
import os
import random
import tempfile
import unittest

import numpy as np

from transformers import TvltFeatureExtractor, is_datasets_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
     import torch

if is_datasets_available():
     from datasets import load_dataset

_snake_case  :    List[Any]						    =  random.Random()

def   a_    ( lowerCAmelCase_      :			List[str],	lowerCAmelCase_      :			List[str]=1.0,	lowerCAmelCase_      :			Dict=None,	lowerCAmelCase_      :			Optional[Any]=None	):
   if rng is None:
      __lowerCAmelCase								=	global_rng

   __lowerCAmelCase								=	[]
   for batch_idx in range(shape[0]	):
      values.append([]	)
      for _ in range(shape[1]	):
         values[-1].append(rng.random() * scale	)

   return values



class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def __init__(				self :		int					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		List[str]=7					,  lowerCAmelCase_ :		int=4_0_0					,  lowerCAmelCase_ :		Any=2_0_0_0					,  lowerCAmelCase_ :		Optional[int]=2_0_4_8					,  lowerCAmelCase_ :		List[str]=1_2_8					,  lowerCAmelCase_ :		List[Any]=1					,  lowerCAmelCase_ :		Optional[Any]=5_1_2					,  lowerCAmelCase_ :		str=3_0					,  lowerCAmelCase_ :		List[Any]=4_4_1_0_0					,  )							->						Optional[Any]:
     __lowerCAmelCase								=	parent
     __lowerCAmelCase								=	batch_size
     __lowerCAmelCase								=	min_seq_length
     __lowerCAmelCase								=	max_seq_length
     __lowerCAmelCase								=	(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
     __lowerCAmelCase								=	spectrogram_length
     __lowerCAmelCase								=	feature_size
     __lowerCAmelCase								=	num_audio_channels
     __lowerCAmelCase								=	hop_length
     __lowerCAmelCase								=	chunk_length
     __lowerCAmelCase								=	sampling_rate






  def 				lowercase		(				self :		List[Any]     )							->						Any:
     return {
         "spectrogram_length": self.spectrogram_length,
         "feature_size": self.feature_size,
         "num_audio_channels": self.num_audio_channels,
         "hop_length": self.hop_length,
         "chunk_length": self.chunk_length,
         "sampling_rate": self.sampling_rate,
     }






  def 				lowercase		(				self :		Union[str, Any]					,  lowerCAmelCase_ :		Optional[Any]=False					,  lowerCAmelCase_ :		Dict=False     )							->						Tuple:
     def _flatten(lowerCAmelCase_ :		Union[str, Any]     ):
        return list(itertools.chain(*__lowerCamelCase     )     )

     if equal_length:
        __lowerCAmelCase								=	[floats_list((self.max_seq_length, self.feature_size)     ) for _ in range(self.batch_size     )]
     else:
        # make sure that inputs increase in size
        __lowerCAmelCase								=	[
            floats_list((x, self.feature_size)     )
            for x in range(self.min_seq_length					,  self.max_seq_length					,  self.seq_length_diff     )
        ]
     if numpify:
        __lowerCAmelCase								=	[np.asarray(__lowerCamelCase     ) for x in speech_inputs]
     return speech_inputs





@require_torch
@require_torchaudio
class 				_UpperCAmelCase    (    UpperCamelCase__		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = TvltFeatureExtractor






  def 				lowercase		(				self :		Dict     )							->						int:
     __lowerCAmelCase								=	TvltFeatureExtractionTester(self     )






  def 				lowercase		(				self :		Dict     )							->						Union[str, Any]:
     __lowerCAmelCase								=	self.feature_extraction_class(**self.feat_extract_dict     )
     self.assertTrue(hasattr(__lowerCamelCase					,  'spectrogram_length'     )     )
     self.assertTrue(hasattr(__lowerCamelCase					,  'feature_size'     )     )
     self.assertTrue(hasattr(__lowerCamelCase					,  'num_audio_channels'     )     )
     self.assertTrue(hasattr(__lowerCamelCase					,  'hop_length'     )     )
     self.assertTrue(hasattr(__lowerCamelCase					,  'chunk_length'     )     )
     self.assertTrue(hasattr(__lowerCamelCase					,  'sampling_rate'     )     )






  def 				lowercase		(				self :		Optional[int]     )							->						Dict:
     __lowerCAmelCase								=	self.feature_extraction_class(**self.feat_extract_dict     )

     with tempfile.TemporaryDirectory() as tmpdirname:
        __lowerCAmelCase								=	feat_extract_first.save_pretrained(__lowerCamelCase     )[0]
        check_json_file_has_correct_format(__lowerCamelCase     )
        __lowerCAmelCase								=	self.feature_extraction_class.from_pretrained(__lowerCamelCase     )

     __lowerCAmelCase								=	feat_extract_first.to_dict()
     __lowerCAmelCase								=	feat_extract_second.to_dict()
     __lowerCAmelCase								=	dict_first.pop('mel_filters'     )
     __lowerCAmelCase								=	dict_second.pop('mel_filters'     )
     self.assertTrue(np.allclose(__lowerCamelCase					,  __lowerCamelCase     )     )
     self.assertEqual(__lowerCamelCase					,  __lowerCamelCase     )






  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     __lowerCAmelCase								=	self.feature_extraction_class(**self.feat_extract_dict     )

     with tempfile.TemporaryDirectory() as tmpdirname:
        __lowerCAmelCase								=	os.path.join(__lowerCamelCase					,  'feat_extract.json'     )
        feat_extract_first.to_json_file(__lowerCamelCase     )
        __lowerCAmelCase								=	self.feature_extraction_class.from_json_file(__lowerCamelCase     )

     __lowerCAmelCase								=	feat_extract_first.to_dict()
     __lowerCAmelCase								=	feat_extract_second.to_dict()
     __lowerCAmelCase								=	dict_first.pop('mel_filters'     )
     __lowerCAmelCase								=	dict_second.pop('mel_filters'     )
     self.assertTrue(np.allclose(__lowerCamelCase					,  __lowerCamelCase     )     )
     self.assertEqual(__lowerCamelCase					,  __lowerCamelCase     )






  def 				lowercase		(				self :		int     )							->						int:
     # Initialize feature_extractor
     __lowerCAmelCase								=	self.feature_extraction_class(**self.feat_extract_dict     )

     # create three inputs of length 800, 1000, and 1200
     __lowerCAmelCase								=	[floats_list((1, x)     )[0] for x in range(8_0_0					,  1_4_0_0					,  2_0_0     )]
     __lowerCAmelCase								=	[np.asarray(__lowerCamelCase     ) for speech_input in speech_inputs]

     # Test not batched input
     __lowerCAmelCase								=	feature_extractor(np_speech_inputs[0]					,  return_tensors='np'					,  sampling_rate=4_4_1_0_0     ).audio_values

     self.assertTrue(encoded_audios.ndim == 4     )
     self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size     )
     self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length     )
     self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels     )

     # Test batched
     __lowerCAmelCase								=	feature_extractor(__lowerCamelCase					,  return_tensors='np'					,  sampling_rate=4_4_1_0_0     ).audio_values

     self.assertTrue(encoded_audios.ndim == 4     )
     self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size     )
     self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length     )
     self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels     )

     # Test audio masking
     __lowerCAmelCase								=	feature_extractor(
         __lowerCamelCase					,  return_tensors='np'					,  sampling_rate=4_4_1_0_0					,  mask_audio=__lowerCamelCase     ).audio_values

     self.assertTrue(encoded_audios.ndim == 4     )
     self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size     )
     self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length     )
     self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels     )

     # Test 2-D numpy arrays are batched.
     __lowerCAmelCase								=	[floats_list((1, x)     )[0] for x in (8_0_0, 8_0_0, 8_0_0)]
     __lowerCAmelCase								=	np.asarray(__lowerCamelCase     )
     __lowerCAmelCase								=	feature_extractor(__lowerCamelCase					,  return_tensors='np'					,  sampling_rate=4_4_1_0_0     ).audio_values
     self.assertTrue(encoded_audios.ndim == 4     )
     self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size     )
     self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length     )
     self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels     )






  def 				lowercase		(				self :		int					,  lowerCAmelCase_ :		List[str]     )							->						Tuple:
     __lowerCAmelCase								=	load_dataset('hf-internal-testing/librispeech_asr_dummy'					,  'clean'					,  split='validation'     )
     # automatic decoding with librispeech
     __lowerCAmelCase								=	ds.sort('id'     ).select(range(__lowerCamelCase     )     )[:num_samples]['audio']

     return [x["array"] for x in speech_samples]






  def 				lowercase		(				self :		List[Any]     )							->						Optional[int]:
     __lowerCAmelCase								=	self._load_datasamples(1     )
     __lowerCAmelCase								=	TvltFeatureExtractor()
     __lowerCAmelCase								=	feature_extractor(__lowerCamelCase					,  return_tensors='pt'     ).audio_values

     self.assertEquals(audio_values.shape					,  (1, 1, 1_9_2, 1_2_8)     )

     __lowerCAmelCase								=	torch.tensor([[-0.30_32, -0.27_08], [-0.44_34, -0.40_07]]     )
     self.assertTrue(torch.allclose(audio_values[0, 0, :2, :2]					,  __lowerCamelCase					,  atol=1e-4     )     )








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


a__:      Dict									=		logging.get_logger(__name__)

a__:      str									=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

a__:      Any									=		{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

a__:      Any									=		{
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


a__:      Optional[Any]									=		{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}




class     SCREAMING_SNAKE_CASE__ (						UpperCamelCase__      ):
 __SCREAMING_SNAKE_CASE							       =	VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_INIT_CONFIGURATION
 __SCREAMING_SNAKE_CASE							       =	['''input_ids''', '''attention_mask''']
 __SCREAMING_SNAKE_CASE							       =	DistilBertTokenizer



 def __init__(							self,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=True,__lowerCamelCase="[UNK]",__lowerCamelCase="[SEP]",__lowerCamelCase="[PAD]",__lowerCamelCase="[CLS]",__lowerCamelCase="[MASK]",__lowerCamelCase=True,__lowerCamelCase=None,**__lowerCamelCase,):
        super().__init__(
            __lowerCamelCase,tokenizer_file=__lowerCamelCase,do_lower_case=__lowerCamelCase,unk_token=__lowerCamelCase,sep_token=__lowerCamelCase,pad_token=__lowerCamelCase,cls_token=__lowerCamelCase,mask_token=__lowerCamelCase,tokenize_chinese_chars=__lowerCamelCase,strip_accents=__lowerCamelCase,**__lowerCamelCase,)

        A__											=						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('''lowercase''',__lowerCamelCase ) != do_lower_case
            or normalizer_state.get('''strip_accents''',__lowerCamelCase ) != strip_accents
            or normalizer_state.get('''handle_chinese_chars''',__lowerCamelCase ) != tokenize_chinese_chars
        ):
               A__											=						getattr(__lowerCamelCase,normalizer_state.pop('''type''' ) )
               A__											=						do_lower_case
               A__											=						strip_accents
               A__											=						tokenize_chinese_chars
               A__											=						normalizer_class(**__lowerCamelCase )

        A__											=						do_lower_case



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase=None ):
        A__											=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase = None ):
        A__											=						[self.sep_token_id]
        A__											=						[self.cls_token_id]
        if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase = None ):
        A__											=						self._tokenizer.model.save(__lowerCamelCase,name=__lowerCamelCase )
        return tuple(__lowerCamelCase )





