

"""simple docstring"""


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A						   =			"▁"

__A						   =			{"vocab_file": "spiece.model"}

__A						   =			{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

__A						   =			{
    "google/pegasus-xsum": 512,
}


__A						   =			logging.get_logger(__name__)





class     snake_case	(				__snake_case  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  VOCAB_FILES_NAMES

				SCREAMING_SNAKE_CASE_   :      Any         =  VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_   :      str         =  PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_   :      Tuple         =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_   :      int         =  ["""input_ids""", """attention_mask"""]

				def __init__(	self							:				int     ,  UpperCamelCase__							:				Tuple     ,  UpperCamelCase__							:				Union[str, Any]="<pad>"     ,  UpperCamelCase__							:				Dict="</s>"     ,  UpperCamelCase__							:				str="<unk>"     ,  UpperCamelCase__							:				int="<mask_2>"     ,  UpperCamelCase__							:				str="<mask_1>"     ,  UpperCamelCase__							:				List[Any]=None     ,  UpperCamelCase__							:				Tuple=1_0_3     ,  UpperCamelCase__							:				Optional[Dict[str, Any]] = None     ,  **UpperCamelCase__							:				Optional[int]     ,  )->   None:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			offset
						if additional_special_tokens is not None:
								if not isinstance(UpperCamelCase__     ,  UpperCamelCase__):
										raise TypeError(
										    f"additional_special_tokens should be of type {type(UpperCamelCase__)}, but is"
										    f" {type(UpperCamelCase__)}")

								__lowerCAmelCase:       str          =			(
								    ([mask_token_sent] + additional_special_tokens)
								    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
								    else additional_special_tokens
								)
								# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
								additional_special_tokens_extended += [
								    f"<unk_{i}>" for i in range(len(UpperCamelCase__)     ,  self.offset - 1)
								]

								if len(set(UpperCamelCase__)) != len(UpperCamelCase__):
										raise ValueError(
										    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
										    f" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.")
								__lowerCAmelCase:       Any          =			additional_special_tokens_extended
						else:
								__lowerCAmelCase:       Any          =			[mask_token_sent] if mask_token_sent is not None else []
								additional_special_tokens += [f"<unk_{i}>" for i in range(2     ,  self.offset)]

						__lowerCAmelCase:       str          =			{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  mask_token=UpperCamelCase__     ,  pad_token=UpperCamelCase__     ,  mask_token_sent=UpperCamelCase__     ,  offset=UpperCamelCase__     ,  additional_special_tokens=UpperCamelCase__     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase__     ,  )
						__lowerCAmelCase:       Optional[int]          =			mask_token_sent
						__lowerCAmelCase:       Optional[Any]          =			vocab_file
						__lowerCAmelCase:       Union[str, Any]          =			spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(UpperCamelCase__)

						# add special tokens to encoder dict
						__lowerCAmelCase:       Dict[int, str]          =			{
						    0: self.pad_token,
						    1: self.eos_token,
						}

						if self.mask_token_sent is not None:
								self.encoder.update(
								    {
								        2: self.mask_token_sent,
								        3: self.mask_token,
								    })

						if self.offset > 0:
								# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
								# mask_token_sent is already added to list -> so start at 1
								self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1     ,  self.offset - 1)})

						__lowerCAmelCase:       Dict[str, int]          =			{v: k for k, v in self.encoder.items()}

				@property
				def 							lowercase_     (	self							:				Any)->   int:



						'''simple docstring'''


						return len(self.sp_model) + self.offset

				def 							lowercase_     (	self							:				Tuple)->   Dict[str, int]:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			{self.convert_ids_to_tokens(UpperCamelCase__): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab

				def __getstate__(	self							:				List[str])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.__dict__.copy()
						__lowerCAmelCase:       Any          =			None
						return state

				def __setstate__(	self							:				Dict     ,  UpperCamelCase__							:				Optional[int])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			d

						# for backward compatibility
						if not hasattr(self     ,  "sp_model_kwargs"):
								__lowerCAmelCase:       Optional[Any]          =			{}

						__lowerCAmelCase:       List[str]          =			spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(self.vocab_file)

				def 							lowercase_     (	self							:				Union[str, Any]     ,  UpperCamelCase__							:				str)->   List[str]:



						'''simple docstring'''


						return self.sp_model.encode(UpperCamelCase__     ,  out_type=UpperCamelCase__)

				def 							lowercase_     (	self							:				List[str]     ,  UpperCamelCase__							:				str)->   int:



						'''simple docstring'''


						if token in self.decoder:
								return self.decoder[token]
						elif token in self.added_tokens_decoder:
								return self.added_tokens_decoder[token]
						__lowerCAmelCase:       Union[str, Any]          =			self.sp_model.piece_to_id(UpperCamelCase__)
						return sp_id + self.offset

				def 							lowercase_     (	self							:				Dict     ,  UpperCamelCase__							:				int)->   str:



						'''simple docstring'''


						if index in self.encoder:
								return self.encoder[index]
						elif index in self.added_tokens_encoder:
								return self.added_tokens_encoder[index]
						else:
								__lowerCAmelCase:       Optional[int]          =			self.sp_model.IdToPiece(index - self.offset)
						return token

				def 							lowercase_     (	self							:				Union[str, Any]     ,  UpperCamelCase__							:				Tuple)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			[]
						__lowerCAmelCase:       Tuple          =			""
						for token in tokens:
								# make sure that special tokens are not decoded using sentencepiece model
								if token in self.all_special_tokens:
										out_string += self.sp_model.decode(UpperCamelCase__) + token
										__lowerCAmelCase:       str          =			[]
								else:
										current_sub_tokens.append(UpperCamelCase__)
						out_string += self.sp_model.decode(UpperCamelCase__)
						return out_string.strip()

				def 							lowercase_     (	self							:				Optional[Any]     ,  UpperCamelCase__							:				Dict=False)->   Any:



						'''simple docstring'''


						return 1

				def 							lowercase_     (	self							:				Tuple     ,  UpperCamelCase__							:				str)->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			set(self.all_special_ids)  # call it once instead of inside list comp
						all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

						return [1 if x in all_special_ids else 0 for x in seq]

				def 							lowercase_     (	self							:				Dict     ,  UpperCamelCase__							:				List     ,  UpperCamelCase__							:				Optional[List] = None     ,  UpperCamelCase__							:				bool = False)->   List[int]:



						'''simple docstring'''


						if already_has_special_tokens:
								return self._special_token_mask(UpperCamelCase__)
						elif token_ids_a is None:
								return self._special_token_mask(UpperCamelCase__) + [1]
						else:
								return self._special_token_mask(token_ids_a + token_ids_a) + [1]

				def 							lowercase_     (	self							:				Optional[Any]     ,  UpperCamelCase__							:				Optional[Any]     ,  UpperCamelCase__							:				Union[str, Any]=None)->   List[int]:



						'''simple docstring'''


						if token_ids_a is None:
								return token_ids_a + [self.eos_token_id]
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return token_ids_a + token_ids_a + [self.eos_token_id]




				def 							lowercase_     (	self							:				List[Any]     ,  UpperCamelCase__							:				str     ,  UpperCamelCase__							:				Optional[str] = None)->   Tuple[str]:



						'''simple docstring'''


						if not os.path.isdir(UpperCamelCase__):
								logger.error(f"Vocabulary path ({save_directory}) should be a directory")
								return
						__lowerCAmelCase:       Optional[int]          =			os.path.join(
						    UpperCamelCase__     ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

						if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCamelCase__) and os.path.isfile(self.vocab_file):
								copyfile(self.vocab_file     ,  UpperCamelCase__)
						elif not os.path.isfile(self.vocab_file):
								with open(UpperCamelCase__     ,  "wb") as fi:
										__lowerCAmelCase:       int          =			self.sp_model.serialized_model_proto()
										fi.write(UpperCamelCase__)

						return (out_vocab_file,)





"""simple docstring"""


import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin





class     snake_case	(				__snake_case  ):
				# to overwrite at feature extractactor specific tests
				SCREAMING_SNAKE_CASE_   :      List[str]         =  None
				SCREAMING_SNAKE_CASE_   :      Tuple         =  None

				@property
				def 							lowercase_     (	self							:				str)->   Any:



						'''simple docstring'''


						return self.feat_extract_tester.prepare_feat_extract_dict()

				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			self.feature_extraction_class(**self.feat_extract_dict)
						self.assertTrue(hasattr(UpperCamelCase__     ,  "feature_size"))
						self.assertTrue(hasattr(UpperCamelCase__     ,  "sampling_rate"))
						self.assertTrue(hasattr(UpperCamelCase__     ,  "padding_value"))

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.feat_extract_tester.prepare_inputs_for_common()
						__lowerCAmelCase:       str          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       int          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       List[Any]          =			BatchFeature({input_name: speech_inputs})

						self.assertTrue(all(len(UpperCamelCase__) == len(UpperCamelCase__) for x, y in zip(UpperCamelCase__     ,  processed_features[input_name])))

						__lowerCAmelCase:       Tuple          =			self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCamelCase__)
						__lowerCAmelCase:       Optional[int]          =			BatchFeature({input_name: speech_inputs}     ,  tensor_type="np")

						__lowerCAmelCase:       Optional[int]          =			processed_features[input_name]

						if len(batch_features_input.shape) < 3:
								__lowerCAmelCase:       Dict          =			batch_features_input[:, :, None]

						self.assertTrue(
						    batch_features_input.shape
						    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]), self.feat_extract_tester.feature_size))

				@require_torch
				def 							lowercase_     (	self							:				Optional[int])->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCamelCase__)
						__lowerCAmelCase:       int          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       Any          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       Any          =			BatchFeature({input_name: speech_inputs}     ,  tensor_type="pt")

						__lowerCAmelCase:       str          =			processed_features[input_name]

						if len(batch_features_input.shape) < 3:
								__lowerCAmelCase:       Any          =			batch_features_input[:, :, None]

						self.assertTrue(
						    batch_features_input.shape
						    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]), self.feat_extract_tester.feature_size))

				@require_tf
				def 							lowercase_     (	self							:				str)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCamelCase__)
						__lowerCAmelCase:       Optional[int]          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       Any          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       Optional[int]          =			BatchFeature({input_name: speech_inputs}     ,  tensor_type="tf")

						__lowerCAmelCase:       int          =			processed_features[input_name]

						if len(batch_features_input.shape) < 3:
								__lowerCAmelCase:       Dict          =			batch_features_input[:, :, None]

						self.assertTrue(
						    batch_features_input.shape
						    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]), self.feat_extract_tester.feature_size))

				def 							lowercase_     (	self							:				Union[str, Any]     ,  UpperCamelCase__							:				List[str]=False)->   Optional[int]:



						'''simple docstring'''


						def _inputs_have_equal_length(UpperCamelCase__							:				Tuple):
								__lowerCAmelCase:       Optional[int]          =			len(input[0])
								for input_slice in input[1:]:
										if len(UpperCamelCase__) != length:
												return False
								return True

						def _inputs_are_equal(UpperCamelCase__							:				str     ,  UpperCamelCase__							:				str):
								if len(UpperCamelCase__) != len(UpperCamelCase__):
										return False

								for input_slice_a, input_slice_a in zip(UpperCamelCase__     ,  UpperCamelCase__):
										if not np.allclose(np.asarray(UpperCamelCase__)     ,  np.asarray(UpperCamelCase__)     ,  atol=1e-3):
												return False
								return True

						__lowerCAmelCase:       List[str]          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       Optional[Any]          =			self.feat_extract_tester.prepare_inputs_for_common(numpify=UpperCamelCase__)
						__lowerCAmelCase:       Tuple          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       List[Any]          =			BatchFeature({input_name: speech_inputs})

						__lowerCAmelCase:       Optional[int]          =			self.feat_extract_tester.seq_length_diff
						__lowerCAmelCase:       str          =			self.feat_extract_tester.max_seq_length + pad_diff
						__lowerCAmelCase:       Tuple          =			self.feat_extract_tester.min_seq_length
						__lowerCAmelCase:       Union[str, Any]          =			self.feat_extract_tester.batch_size
						__lowerCAmelCase:       Tuple          =			self.feat_extract_tester.feature_size

						# test padding for List[int] + numpy
						__lowerCAmelCase:       Optional[int]          =			feat_extract.pad(UpperCamelCase__     ,  padding=UpperCamelCase__)
						__lowerCAmelCase:       Optional[int]          =			input_a[input_name]

						__lowerCAmelCase:       List[Any]          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest")
						__lowerCAmelCase:       int          =			input_a[input_name]

						__lowerCAmelCase:       Tuple          =			feat_extract.pad(UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[-1]))
						__lowerCAmelCase:       Optional[Any]          =			input_a[input_name]

						__lowerCAmelCase:       Union[str, Any]          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  return_tensors="np")
						__lowerCAmelCase:       Optional[int]          =			input_a[input_name]

						# max_length parameter has to be provided when setting `padding="max_length"`
						with self.assertRaises(UpperCamelCase__):
								feat_extract.pad(UpperCamelCase__     ,  padding="max_length")[input_name]

						__lowerCAmelCase:       Any          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=UpperCamelCase__     ,  return_tensors="np")
						__lowerCAmelCase:       Optional[Any]          =			input_a[input_name]

						self.assertFalse(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(_inputs_are_equal(UpperCamelCase__     ,  UpperCamelCase__))
						self.assertTrue(len(input_a[0]) == pad_min_length)
						self.assertTrue(len(input_a[1]) == pad_min_length + pad_diff)
						self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0])))
						self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length))

						if feature_size > 1:
								self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size)

						# test padding for `pad_to_multiple_of` for List[int] + numpy
						__lowerCAmelCase:       Dict          =			feat_extract.pad(UpperCamelCase__     ,  pad_to_multiple_of=1_0)
						__lowerCAmelCase:       Tuple          =			input_a[input_name]

						__lowerCAmelCase:       Tuple          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  pad_to_multiple_of=1_0)
						__lowerCAmelCase:       List[str]          =			input_a[input_name]

						__lowerCAmelCase:       str          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  pad_to_multiple_of=1_0     ,  max_length=UpperCamelCase__)
						__lowerCAmelCase:       str          =			input_a[input_name]

						__lowerCAmelCase:       Dict          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  pad_to_multiple_of=1_0     ,  max_length=UpperCamelCase__     ,  return_tensors="np"     ,  )
						__lowerCAmelCase:       List[str]          =			input_a[input_name]

						self.assertTrue(all(len(UpperCamelCase__) % 1_0 == 0 for x in input_a))
						self.assertTrue(_inputs_are_equal(UpperCamelCase__     ,  UpperCamelCase__))

						__lowerCAmelCase:       Optional[Any]          =			pad_max_length if pad_max_length % 1_0 == 0 else (pad_max_length // 1_0 + 1) * 1_0
						self.assertTrue(all(len(UpperCamelCase__) == expected_mult_pad_length for x in input_a))
						self.assertEqual(input_a.shape[:2]     ,  (batch_size, expected_mult_pad_length))

						if feature_size > 1:
								self.assertTrue(input_a.shape[2] == feature_size)

						# Check padding value is correct
						__lowerCAmelCase:       Any          =			(np.ones(self.feat_extract_tester.feature_size) * feat_extract.padding_value).sum()
						self.assertTrue(
						    abs(np.asarray(input_a[0])[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length))
						    < 1e-3)
						self.assertTrue(
						    abs(
						        np.asarray(input_a[1])[pad_min_length + pad_diff :].sum()
						        - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff))
						    < 1e-3)
						self.assertTrue(
						    abs(
						        np.asarray(input_a[2])[pad_min_length + 2 * pad_diff :].sum()
						        - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff))
						    < 1e-3)
						self.assertTrue(
						    abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)) < 1e-3)
						self.assertTrue(
						    abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length))
						    < 1e-3)

				def 							lowercase_     (	self							:				Optional[int]     ,  UpperCamelCase__							:				Optional[int]=False)->   str:



						'''simple docstring'''


						def _inputs_have_equal_length(UpperCamelCase__							:				List[Any]):
								__lowerCAmelCase:       List[str]          =			len(input[0])
								for input_slice in input[1:]:
										if len(UpperCamelCase__) != length:
												return False
								return True

						def _inputs_are_equal(UpperCamelCase__							:				Optional[int]     ,  UpperCamelCase__							:				List[Any]):
								if len(UpperCamelCase__) != len(UpperCamelCase__):
										return False

								for input_slice_a, input_slice_a in zip(UpperCamelCase__     ,  UpperCamelCase__):
										if not np.allclose(np.asarray(UpperCamelCase__)     ,  np.asarray(UpperCamelCase__)     ,  atol=1e-3):
												return False
								return True

						__lowerCAmelCase:       Dict          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       Any          =			self.feat_extract_tester.prepare_inputs_for_common(numpify=UpperCamelCase__)
						__lowerCAmelCase:       Any          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       Optional[int]          =			BatchFeature({input_name: speech_inputs})

						# truncate to smallest
						__lowerCAmelCase:       int          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[0])     ,  truncation=UpperCamelCase__)
						__lowerCAmelCase:       Any          =			input_a[input_name]

						__lowerCAmelCase:       str          =			feat_extract.pad(UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[0]))
						__lowerCAmelCase:       Tuple          =			input_a[input_name]

						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertFalse(_inputs_have_equal_length(UpperCamelCase__))

						# truncate to smallest with np
						__lowerCAmelCase:       Optional[int]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[0])     ,  return_tensors="np"     ,  truncation=UpperCamelCase__     ,  )
						__lowerCAmelCase:       List[str]          =			input_a[input_name]

						__lowerCAmelCase:       List[str]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[0])     ,  return_tensors="np")
						__lowerCAmelCase:       Any          =			input_a[input_name]

						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(input_a.shape[1] == len(speech_inputs[0]))

						# since truncation forces padding to be smaller than longest input
						# function can't return `np.ndarray`, but has to return list
						self.assertFalse(_inputs_have_equal_length(UpperCamelCase__))

						# truncate to middle
						__lowerCAmelCase:       Union[str, Any]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[1])     ,  truncation=UpperCamelCase__     ,  return_tensors="np"     ,  )
						__lowerCAmelCase:       int          =			input_a[input_name]

						__lowerCAmelCase:       List[Any]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[1])     ,  truncation=UpperCamelCase__)
						__lowerCAmelCase:       Dict          =			input_a[input_name]

						__lowerCAmelCase:       List[Any]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[1])     ,  return_tensors="np")
						__lowerCAmelCase:       str          =			input_a[input_name]

						self.assertTrue(input_a.shape[1] == len(speech_inputs[1]))
						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(_inputs_are_equal(UpperCamelCase__     ,  UpperCamelCase__))

						# since truncation forces padding to be smaller than longest input
						# function can't return `np.ndarray`, but has to return list
						self.assertFalse(_inputs_have_equal_length(UpperCamelCase__))
						self.assertTrue(len(input_a[-1]) == len(speech_inputs[-1]))

						# padding has to be max_length when setting `truncation=True`
						with self.assertRaises(UpperCamelCase__):
								feat_extract.pad(UpperCamelCase__     ,  truncation=UpperCamelCase__)[input_name]

						# padding has to be max_length when setting `truncation=True`
						with self.assertRaises(UpperCamelCase__):
								feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  truncation=UpperCamelCase__)[input_name]

						# padding has to be max_length when setting `truncation=True`
						with self.assertRaises(UpperCamelCase__):
								feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  truncation=UpperCamelCase__)[input_name]

						# max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
						with self.assertRaises(UpperCamelCase__):
								feat_extract.pad(UpperCamelCase__     ,  padding="max_length"     ,  truncation=UpperCamelCase__)[input_name]

						# test truncation for `pad_to_multiple_of` for List[int] + numpy
						__lowerCAmelCase:       Tuple          =			1_2
						__lowerCAmelCase:       Optional[int]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[0])     ,  pad_to_multiple_of=UpperCamelCase__     ,  truncation=UpperCamelCase__     ,  )
						__lowerCAmelCase:       Dict          =			input_a[input_name]

						__lowerCAmelCase:       List[str]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=len(speech_inputs[0])     ,  pad_to_multiple_of=UpperCamelCase__     ,  )
						__lowerCAmelCase:       str          =			input_a[input_name]

						# retrieve expected_length as multiple of pad_to_multiple_of
						__lowerCAmelCase:       Optional[Any]          =			len(speech_inputs[0])
						if expected_length % pad_to_multiple_of != 0:
								__lowerCAmelCase:       Optional[Any]          =			((len(speech_inputs[0]) // pad_to_multiple_of) + 1) * pad_to_multiple_of

						self.assertTrue(len(input_a[0]) == expected_length)
						self.assertTrue(_inputs_have_equal_length(UpperCamelCase__))
						self.assertFalse(_inputs_have_equal_length(UpperCamelCase__))

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[Any]:



						'''simple docstring'''


						self._check_padding(numpify=UpperCamelCase__)

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						self._check_padding(numpify=UpperCamelCase__)

				def 							lowercase_     (	self							:				List[Any])->   Tuple:



						'''simple docstring'''


						self._check_truncation(numpify=UpperCamelCase__)

				def 							lowercase_     (	self							:				Optional[Any])->   Optional[Any]:



						'''simple docstring'''


						self._check_truncation(numpify=UpperCamelCase__)

				@require_torch
				def 							lowercase_     (	self							:				Dict)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       Optional[Any]          =			self.feat_extract_tester.prepare_inputs_for_common()
						__lowerCAmelCase:       Any          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       str          =			BatchFeature({input_name: speech_inputs})

						__lowerCAmelCase:       str          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  return_tensors="np")[input_name]
						__lowerCAmelCase:       Dict          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  return_tensors="pt")[input_name]

						self.assertTrue(abs(input_np.astype(np.floataa).sum() - input_pt.numpy().astype(np.floataa).sum()) < 1e-2)

				@require_tf
				def 							lowercase_     (	self							:				Tuple)->   Union[str, Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Dict          =			self.feature_extraction_class(**self.feat_extract_dict)
						__lowerCAmelCase:       Tuple          =			self.feat_extract_tester.prepare_inputs_for_common()
						__lowerCAmelCase:       List[Any]          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       Any          =			BatchFeature({input_name: speech_inputs})

						__lowerCAmelCase:       Dict          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  return_tensors="np")[input_name]
						__lowerCAmelCase:       Any          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  return_tensors="tf")[input_name]

						self.assertTrue(abs(input_np.astype(np.floataa).sum() - input_tf.numpy().astype(np.floataa).sum()) < 1e-2)

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.feat_extract_dict
						__lowerCAmelCase:       Any          =			True
						__lowerCAmelCase:       str          =			self.feature_extraction_class(**UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			self.feat_extract_tester.prepare_inputs_for_common()
						__lowerCAmelCase:       Tuple          =			[len(UpperCamelCase__) for x in speech_inputs]
						__lowerCAmelCase:       Optional[Any]          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       Optional[int]          =			BatchFeature({input_name: speech_inputs})

						__lowerCAmelCase:       Optional[Any]          =			feat_extract.pad(UpperCamelCase__     ,  padding="longest"     ,  return_tensors="np")
						self.assertIn("attention_mask"     ,  UpperCamelCase__)
						self.assertListEqual(list(processed.attention_mask.shape)     ,  list(processed[input_name].shape[:2]))
						self.assertListEqual(processed.attention_mask.sum(-1).tolist()     ,  UpperCamelCase__)




				def 							lowercase_     (	self							:				List[str])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.feat_extract_dict
						__lowerCAmelCase:       str          =			True
						__lowerCAmelCase:       int          =			self.feature_extraction_class(**UpperCamelCase__)
						__lowerCAmelCase:       int          =			self.feat_extract_tester.prepare_inputs_for_common()
						__lowerCAmelCase:       str          =			[len(UpperCamelCase__) for x in speech_inputs]
						__lowerCAmelCase:       Any          =			feat_extract.model_input_names[0]

						__lowerCAmelCase:       Optional[int]          =			BatchFeature({input_name: speech_inputs})
						__lowerCAmelCase:       List[str]          =			min(UpperCamelCase__)

						__lowerCAmelCase:       List[str]          =			feat_extract.pad(
						    UpperCamelCase__     ,  padding="max_length"     ,  max_length=UpperCamelCase__     ,  truncation=UpperCamelCase__     ,  return_tensors="np")
						self.assertIn("attention_mask"     ,  UpperCamelCase__)
						self.assertListEqual(
						    list(processed_pad.attention_mask.shape)     ,  [processed_pad[input_name].shape[0], max_length])
						self.assertListEqual(
						    processed_pad.attention_mask[:, :max_length].sum(-1).tolist()     ,  [max_length for x in speech_inputs])


