
import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


__snake_case								=  logging.get_logger(__name__)




class 						lowercase__  (						_UpperCAmelCase  ):
	A__    :   Dict				       =["""input_features"""]
	def __init__(			self							:   Tuple       ,   UpperCAmelCase_							:   Any=80       ,   UpperCAmelCase_							:   int=16000       ,   UpperCAmelCase_							:   Dict=160       ,   UpperCAmelCase_							:   Tuple=30       ,   UpperCAmelCase_							:   List[Any]=400       ,   UpperCAmelCase_							:   int=0.0       ,   UpperCAmelCase_							:   Any=False       ,   **UpperCAmelCase_							:   Dict       ,   ):
						super().__init__(
						    feature_size=UpperCAmelCase_       ,   sampling_rate=UpperCAmelCase_       ,   padding_value=UpperCAmelCase_       ,   return_attention_mask=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )
						SCREAMING_SNAKE_CASE__													= n_fft
						SCREAMING_SNAKE_CASE__													= hop_length
						SCREAMING_SNAKE_CASE__													= chunk_length
						SCREAMING_SNAKE_CASE__													= chunk_length * sampling_rate
						SCREAMING_SNAKE_CASE__													= self.n_samples // hop_length
						SCREAMING_SNAKE_CASE__													= sampling_rate
						SCREAMING_SNAKE_CASE__													= mel_filter_bank(
						    num_frequency_bins=1 + n_fft // 2       ,   num_mel_filters=UpperCAmelCase_       ,   min_frequency=0.0       ,   max_frequency=8_000.0       ,   sampling_rate=UpperCAmelCase_       ,   norm='slaney'       ,   mel_scale='slaney'       ,   )
	def      A_				(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   np.array							):
						SCREAMING_SNAKE_CASE__													= spectrogram(
						    UpperCAmelCase_       ,   window_function(self.n_fft       ,   'hann'							)       ,   frame_length=self.n_fft       ,   hop_length=self.hop_length       ,   power=2.0       ,   mel_filters=self.mel_filters       ,   log_mel='log10'       ,   )
						SCREAMING_SNAKE_CASE__													= log_spec[:, :-1]
						SCREAMING_SNAKE_CASE__													= np.maximum(UpperCAmelCase_       ,   log_spec.max() - 8.0							)
						SCREAMING_SNAKE_CASE__													= (log_spec + 4.0) / 4.0
						return log_spec
	@staticmethod
	# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
	def      A_				(			UpperCAmelCase_							:   List[np.ndarray]       ,   UpperCAmelCase_							:   List[np.ndarray]       ,   UpperCAmelCase_							:   float = 0.0							):
						if attention_mask is not None:
											SCREAMING_SNAKE_CASE__													= np.array(UpperCAmelCase_       ,   np.intaa							)
											SCREAMING_SNAKE_CASE__													= []

											for vector, length in zip(UpperCAmelCase_       ,   attention_mask.sum(-1							)							):
																SCREAMING_SNAKE_CASE__													= (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7							)
																if length < normed_slice.shape[0]:
																					SCREAMING_SNAKE_CASE__													= padding_value

																normed_input_values.append(UpperCAmelCase_							)
						else:
											SCREAMING_SNAKE_CASE__													= [(x - x.mean()) / np.sqrt(x.var() + 1e-7							) for x in input_values]

						return normed_input_values
	def __call__(			self							:   Tuple       ,   UpperCAmelCase_							:   Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]       ,   UpperCAmelCase_							:   bool = True       ,   UpperCAmelCase_							:   Optional[int] = None       ,   UpperCAmelCase_							:   Optional[Union[str, TensorType]] = None       ,   UpperCAmelCase_							:   Optional[bool] = None       ,   UpperCAmelCase_							:   Optional[str] = "max_length"       ,   UpperCAmelCase_							:   Optional[int] = None       ,   UpperCAmelCase_							:   Optional[int] = None       ,   UpperCAmelCase_							:   Optional[bool] = None       ,   **UpperCAmelCase_							:   int       ,   ):

						if sampling_rate is not None:
											if sampling_rate != self.sampling_rate:
																raise ValueError(
																    F'The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a'
																    F' sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input'
																    F' was sampled with {self.sampling_rate} and not {sampling_rate}.'							)
						else:
											logger.warning(
											    'It is strongly recommended to pass the `sampling_rate` argument to this function. '
											    'Failing to do so can result in silent errors that might be hard to debug.'							)

						SCREAMING_SNAKE_CASE__													= isinstance(UpperCAmelCase_       ,   np.ndarray							) and len(raw_speech.shape							) > 1
						if is_batched_numpy and len(raw_speech.shape							) > 2:
											raise ValueError(F'Only mono-channel audio is supported for input to {self}'							)
						SCREAMING_SNAKE_CASE__													= is_batched_numpy or (
						    isinstance(UpperCAmelCase_       ,   (list, tuple)							) and (isinstance(raw_speech[0]       ,   (np.ndarray, tuple, list)							))
						)

						if is_batched:
											SCREAMING_SNAKE_CASE__													= [np.asarray([speech]       ,   dtype=np.floataa							).T for speech in raw_speech]
						elif not is_batched and not isinstance(UpperCAmelCase_       ,   np.ndarray							):
											SCREAMING_SNAKE_CASE__													= np.asarray(UpperCAmelCase_       ,   dtype=np.floataa							)
						elif isinstance(UpperCAmelCase_       ,   np.ndarray							) and raw_speech.dtype is np.dtype(np.floataa							):
											SCREAMING_SNAKE_CASE__													= raw_speech.astype(np.floataa							)

						# always return batch
						if not is_batched:
											SCREAMING_SNAKE_CASE__													= [np.asarray([raw_speech]							).T]

						SCREAMING_SNAKE_CASE__													= BatchFeature({'input_features': raw_speech}							)

						# convert into correct format for padding

						SCREAMING_SNAKE_CASE__													= self.pad(
						    UpperCAmelCase_       ,   padding=UpperCAmelCase_       ,   max_length=max_length if max_length else self.n_samples       ,   truncation=UpperCAmelCase_       ,   pad_to_multiple_of=UpperCAmelCase_       ,   return_attention_mask=return_attention_mask or do_normalize       ,   )

						# zero-mean and unit-variance normalization
						if do_normalize:
											SCREAMING_SNAKE_CASE__													= self.zero_mean_unit_var_norm(
											    padded_inputs['input_features']       ,   attention_mask=padded_inputs['attention_mask']       ,   padding_value=self.padding_value       ,   )
											SCREAMING_SNAKE_CASE__													= np.stack(padded_inputs['input_features']       ,   axis=0							)

						# make sure list is in array format
						SCREAMING_SNAKE_CASE__													= padded_inputs.get('input_features'							).transpose(2       ,   0       ,   1							)

						SCREAMING_SNAKE_CASE__													= [self._np_extract_fbank_features(UpperCAmelCase_							) for waveform in input_features[0]]

						if isinstance(input_features[0]       ,   UpperCAmelCase_							):
											SCREAMING_SNAKE_CASE__													= [np.asarray(UpperCAmelCase_       ,   dtype=np.floataa							) for feature in input_features]
						else:
											SCREAMING_SNAKE_CASE__													= input_features

						if return_attention_mask:
											# rescale from sample (48000) to feature (3000)
											SCREAMING_SNAKE_CASE__													= padded_inputs['attention_mask'][:, :: self.hop_length]

						if return_tensors is not None:
											SCREAMING_SNAKE_CASE__													= padded_inputs.convert_to_tensors(UpperCAmelCase_							)

						return padded_inputs





	def      A_				(			self							:   Optional[Any]							):
						SCREAMING_SNAKE_CASE__													= copy.deepcopy(self.__dict__							)
						SCREAMING_SNAKE_CASE__													= self.__class__.__name__
						if "mel_filters" in output:
											del output["mel_filters"]
						return output







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__snake_case								=  logging.get_logger(__name__)


__snake_case								=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
__snake_case								=  {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

__snake_case								=  {
    """facebook/bart-base""": 10_24,
    """facebook/bart-large""": 10_24,
    """facebook/bart-large-mnli""": 10_24,
    """facebook/bart-large-cnn""": 10_24,
    """facebook/bart-large-xsum""": 10_24,
    """yjernite/bart_eli5""": 10_24,
}




class 						lowercase__  (						_UpperCAmelCase  ):
	A__    :   Tuple				       =VOCAB_FILES_NAMES
	A__    :   Any				       =PRETRAINED_VOCAB_FILES_MAP
	A__    :   str				       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A__    :   Tuple				       =["""input_ids""", """attention_mask"""]
	A__    :   Optional[int]				       =BartTokenizer
	def __init__(			self							:   str       ,   UpperCAmelCase_							:   Optional[Any]=None       ,   UpperCAmelCase_							:   int=None       ,   UpperCAmelCase_							:   str=None       ,   UpperCAmelCase_							:   Optional[Any]="replace"       ,   UpperCAmelCase_							:   int="<s>"       ,   UpperCAmelCase_							:   List[Any]="</s>"       ,   UpperCAmelCase_							:   int="</s>"       ,   UpperCAmelCase_							:   Tuple="<s>"       ,   UpperCAmelCase_							:   Any="<unk>"       ,   UpperCAmelCase_							:   Any="<pad>"       ,   UpperCAmelCase_							:   Optional[Any]="<mask>"       ,   UpperCAmelCase_							:   Optional[int]=False       ,   UpperCAmelCase_							:   List[str]=True       ,   **UpperCAmelCase_							:   List[Any]       ,   ):
						super().__init__(
						    UpperCAmelCase_       ,   UpperCAmelCase_       ,   tokenizer_file=UpperCAmelCase_       ,   errors=UpperCAmelCase_       ,   bos_token=UpperCAmelCase_       ,   eos_token=UpperCAmelCase_       ,   sep_token=UpperCAmelCase_       ,   cls_token=UpperCAmelCase_       ,   unk_token=UpperCAmelCase_       ,   pad_token=UpperCAmelCase_       ,   mask_token=UpperCAmelCase_       ,   add_prefix_space=UpperCAmelCase_       ,   trim_offsets=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )

						SCREAMING_SNAKE_CASE__													= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
						if pre_tok_state.get('add_prefix_space'       ,   UpperCAmelCase_							) != add_prefix_space:
											SCREAMING_SNAKE_CASE__													= getattr(UpperCAmelCase_       ,   pre_tok_state.pop('type'							)							)
											SCREAMING_SNAKE_CASE__													= add_prefix_space
											SCREAMING_SNAKE_CASE__													= pre_tok_class(**UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						SCREAMING_SNAKE_CASE__													= 'post_processor'
						SCREAMING_SNAKE_CASE__													= getattr(self.backend_tokenizer       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
						if tokenizer_component_instance:
											SCREAMING_SNAKE_CASE__													= json.loads(tokenizer_component_instance.__getstate__()							)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
																SCREAMING_SNAKE_CASE__													= tuple(state['sep']							)
											if "cls" in state:
																SCREAMING_SNAKE_CASE__													= tuple(state['cls']							)

											SCREAMING_SNAKE_CASE__													= False

											if state.get('add_prefix_space'       ,   UpperCAmelCase_							) != add_prefix_space:
																SCREAMING_SNAKE_CASE__													= add_prefix_space
																SCREAMING_SNAKE_CASE__													= True

											if state.get('trim_offsets'       ,   UpperCAmelCase_							) != trim_offsets:
																SCREAMING_SNAKE_CASE__													= trim_offsets
																SCREAMING_SNAKE_CASE__													= True

											if changes_to_apply:
																SCREAMING_SNAKE_CASE__													= getattr(UpperCAmelCase_       ,   state.pop('type'							)							)
																SCREAMING_SNAKE_CASE__													= component_class(**UpperCAmelCase_							)
																setattr(self.backend_tokenizer       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
	@property
	def      A_				(			self							:   Tuple							):
						if self._mask_token is None:
											if self.verbose:
																logger.error('Using mask_token, but it is not set yet.'							)
											return None
						return str(self._mask_token							)
	@mask_token.setter
	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= AddedToken(UpperCAmelCase_       ,   lstrip=UpperCAmelCase_       ,   rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_       ,   UpperCAmelCase_							) else value
						SCREAMING_SNAKE_CASE__													= value
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   Tuple       ,   **UpperCAmelCase_							:   str							):
						SCREAMING_SNAKE_CASE__													= kwargs.get('is_split_into_words'       ,   UpperCAmelCase_							)

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'							)

						return super()._batch_encode_plus(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   Dict       ,   **UpperCAmelCase_							:   int							):
						SCREAMING_SNAKE_CASE__													= kwargs.get('is_split_into_words'       ,   UpperCAmelCase_							)

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'							)

						return super()._encode_plus(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   Optional[str] = None							):
						SCREAMING_SNAKE_CASE__													= self._tokenizer.model.save(UpperCAmelCase_       ,   name=UpperCAmelCase_							)
						return tuple(UpperCAmelCase_							)
	def      A_				(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Union[str, Any]=None							):
						SCREAMING_SNAKE_CASE__													= [self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
											return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   List[int]       ,   UpperCAmelCase_							:   Optional[List[int]] = None							):
						SCREAMING_SNAKE_CASE__													= [self.sep_token_id]
						SCREAMING_SNAKE_CASE__													= [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]





