




import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


lowercase__				:     List[str] 		=						logging.get_logger(__name__)
class 		UpperCAmelCase   (				UpperCAmelCase__			):







						'''simple docstring'''





						lowerCAmelCase_     		=   ['''input_features''']






						def __init__(						self							:     Optional[Any]       ,	__lowercase							:     Optional[int]=80       ,	__lowercase							:     Any=1_60_00       ,	__lowercase							:     Any=1_60       ,	__lowercase							:     Dict=30       ,	__lowercase							:     Union[str, Any]=4_00       ,	__lowercase							:     Tuple=0.0       ,	__lowercase							:     int=False       ,	**__lowercase							:     int       ,	):






							"""simple docstring"""






							super().__init__(
							    feature_size=__lowercase       ,	sampling_rate=__lowercase       ,	padding_value=__lowercase       ,	return_attention_mask=__lowercase       ,	**__lowercase       ,	)
							snake_case_		    =     n_fft
							snake_case_		    =     hop_length
							snake_case_		    =     chunk_length
							snake_case_		    =     chunk_length * sampling_rate
							snake_case_		    =     self.n_samples // hop_length
							snake_case_		    =     sampling_rate
							snake_case_		    =     mel_filter_bank(
							    num_frequency_bins=1 + n_fft // 2       ,	num_mel_filters=__lowercase       ,	min_frequency=0.0       ,	max_frequency=8000.0       ,	sampling_rate=__lowercase       ,	norm="slaney"       ,	mel_scale="slaney"       ,	)






						def        snake_case__						(						self							:     Union[str, Any]       ,	__lowercase							:     np.array						):






							"""simple docstring"""






							snake_case_		    =     spectrogram(
							    __lowercase       ,	window_function(self.n_fft       ,	"hann"						)       ,	frame_length=self.n_fft       ,	hop_length=self.hop_length       ,	power=2.0       ,	mel_filters=self.mel_filters       ,	log_mel="log10"       ,	)
							snake_case_		    =     log_spec[:, :-1]
							snake_case_		    =     np.maximum(__lowercase       ,	log_spec.max() - 8.0						)
							snake_case_		    =     (log_spec + 4.0) / 4.0
							return log_spec






						@staticmethod
						# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
						def        snake_case__						(						__lowercase							:     List[np.ndarray]       ,	__lowercase							:     List[np.ndarray]       ,	__lowercase							:     float = 0.0						):






							"""simple docstring"""






							if attention_mask is not None:
								snake_case_		    =     np.array(__lowercase       ,	np.intaa						)
								snake_case_		    =     []

								for vector, length in zip(__lowercase       ,	attention_mask.sum(-1						)						):
									snake_case_		    =     (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7						)
									if length < normed_slice.shape[0]:
										snake_case_		    =     padding_value

									normed_input_values.append(__lowercase						)
							else:
								snake_case_		    =     [(x - x.mean()) / np.sqrt(x.var() + 1E-7						) for x in input_values]

							return normed_input_values






						def __call__(						self							:     str       ,	__lowercase							:     Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]       ,	__lowercase							:     bool = True       ,	__lowercase							:     Optional[int] = None       ,	__lowercase							:     Optional[Union[str, TensorType]] = None       ,	__lowercase							:     Optional[bool] = None       ,	__lowercase							:     Optional[str] = "max_length"       ,	__lowercase							:     Optional[int] = None       ,	__lowercase							:     Optional[int] = None       ,	__lowercase							:     Optional[bool] = None       ,	**__lowercase							:     Optional[Any]       ,	):






							"""simple docstring"""






							if sampling_rate is not None:
								if sampling_rate != self.sampling_rate:
									raise ValueError(
									    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
									    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
									    f" was sampled with {self.sampling_rate} and not {sampling_rate}."						)
							else:
								logger.warning(
								    "It is strongly recommended to pass the `sampling_rate` argument to this function. "
								    "Failing to do so can result in silent errors that might be hard to debug."						)

							snake_case_		    =     isinstance(__lowercase       ,	np.ndarray						) and len(raw_speech.shape						) > 1
							if is_batched_numpy and len(raw_speech.shape						) > 2:
								raise ValueError(f"Only mono-channel audio is supported for input to {self}"						)
							snake_case_		    =     is_batched_numpy or (
							    isinstance(__lowercase       ,	(list, tuple)						) and (isinstance(raw_speech[0]       ,	(np.ndarray, tuple, list)						))
							)

							if is_batched:
								snake_case_		    =     [np.asarray([speech]       ,	dtype=np.floataa						).T for speech in raw_speech]
							elif not is_batched and not isinstance(__lowercase       ,	np.ndarray						):
								snake_case_		    =     np.asarray(__lowercase       ,	dtype=np.floataa						)
							elif isinstance(__lowercase       ,	np.ndarray						) and raw_speech.dtype is np.dtype(np.floataa						):
								snake_case_		    =     raw_speech.astype(np.floataa						)

							# always return batch
							if not is_batched:
								snake_case_		    =     [np.asarray([raw_speech]						).T]

							snake_case_		    =     BatchFeature({"input_features": raw_speech}						)

							# convert into correct format for padding

							snake_case_		    =     self.pad(
							    __lowercase       ,	padding=__lowercase       ,	max_length=max_length if max_length else self.n_samples       ,	truncation=__lowercase       ,	pad_to_multiple_of=__lowercase       ,	return_attention_mask=return_attention_mask or do_normalize       ,	)

							# zero-mean and unit-variance normalization
							if do_normalize:
								snake_case_		    =     self.zero_mean_unit_var_norm(
								    padded_inputs["input_features"]       ,	attention_mask=padded_inputs["attention_mask"]       ,	padding_value=self.padding_value       ,	)
								snake_case_		    =     np.stack(padded_inputs["input_features"]       ,	axis=0						)

							# make sure list is in array format
							snake_case_		    =     padded_inputs.get("input_features"						).transpose(2       ,	0       ,	1						)

							snake_case_		    =     [self._np_extract_fbank_features(__lowercase						) for waveform in input_features[0]]

							if isinstance(input_features[0]       ,	__lowercase						):
								snake_case_		    =     [np.asarray(__lowercase       ,	dtype=np.floataa						) for feature in input_features]
							else:
								snake_case_		    =     input_features

							if return_attention_mask:
								# rescale from sample (48000) to feature (3000)
								snake_case_		    =     padded_inputs["attention_mask"][:, :: self.hop_length]

							if return_tensors is not None:
								snake_case_		    =     padded_inputs.convert_to_tensors(__lowercase						)

							return padded_inputs






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     copy.deepcopy(self.__dict__						)
							snake_case_		    =     self.__class__.__name__
							if "mel_filters" in output:
								del output["mel_filters"]
							return output






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		UpperCAmelCase   (				UpperCAmelCase__       ,   unittest.TestCase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   CodeGenTokenizer
						lowerCAmelCase_     		=   CodeGenTokenizerFast
						lowerCAmelCase_     		=   True
						lowerCAmelCase_     		=   {'''add_prefix_space''': True}
						lowerCAmelCase_     		=   False






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							snake_case_		    =     [
							    "l",
							    "o",
							    "w",
							    "e",
							    "r",
							    "s",
							    "t",
							    "i",
							    "d",
							    "n",
							    "\u0120",
							    "\u0120l",
							    "\u0120n",
							    "\u0120lo",
							    "\u0120low",
							    "er",
							    "\u0120lowest",
							    "\u0120newer",
							    "\u0120wider",
							    "<unk>",
							    "<|endoftext|>",
							]
							snake_case_		    =     dict(zip(__lowercase       ,	range(len(__lowercase						)						)						)						)
							snake_case_		    =     ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
							snake_case_		    =     {"unk_token": "<unk>"}

							snake_case_		    =     os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["vocab_file"]						)
							snake_case_		    =     os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["merges_file"]						)
							with open(self.vocab_file       ,	"w"       ,	encoding="utf-8"						) as fp:
								fp.write(json.dumps(__lowercase						) + "\n"						)
							with open(self.merges_file       ,	"w"       ,	encoding="utf-8"						) as fp:
								fp.write("\n".join(__lowercase						)						)






						def        snake_case__						(						self							:     Union[str, Any]       ,	**__lowercase							:     List[str]						):






							"""simple docstring"""






							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,	**__lowercase						)






						def        snake_case__						(						self							:     Optional[Any]       ,	**__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,	**__lowercase						)






						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     "lower newer"
							snake_case_		    =     "lower newer"
							return input_text, output_text






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     CodeGenTokenizer(self.vocab_file       ,	self.merges_file       ,	**self.special_tokens_map						)
							snake_case_		    =     "lower newer"
							snake_case_		    =     ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
							snake_case_		    =     tokenizer.tokenize(__lowercase       ,	add_prefix_space=__lowercase						)
							self.assertListEqual(__lowercase       ,	__lowercase						)

							snake_case_		    =     tokens + [tokenizer.unk_token]
							snake_case_		    =     [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						)       ,	__lowercase						)






						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							if not self.test_rust_tokenizer:
								return

							snake_case_		    =     self.get_tokenizer()
							snake_case_		    =     self.get_rust_tokenizer(add_prefix_space=__lowercase						)

							snake_case_		    =     "lower newer"

							# Testing tokenization
							snake_case_		    =     tokenizer.tokenize(__lowercase       ,	add_prefix_space=__lowercase						)
							snake_case_		    =     rust_tokenizer.tokenize(__lowercase						)
							self.assertListEqual(__lowercase       ,	__lowercase						)

							# Testing conversion to ids without special tokens
							snake_case_		    =     tokenizer.encode(__lowercase       ,	add_special_tokens=__lowercase       ,	add_prefix_space=__lowercase						)
							snake_case_		    =     rust_tokenizer.encode(__lowercase       ,	add_special_tokens=__lowercase						)
							self.assertListEqual(__lowercase       ,	__lowercase						)

							# Testing conversion to ids with special tokens
							snake_case_		    =     self.get_rust_tokenizer(add_prefix_space=__lowercase						)
							snake_case_		    =     tokenizer.encode(__lowercase       ,	add_prefix_space=__lowercase						)
							snake_case_		    =     rust_tokenizer.encode(__lowercase						)
							self.assertListEqual(__lowercase       ,	__lowercase						)

							# Testing the unknown token
							snake_case_		    =     tokens + [rust_tokenizer.unk_token]
							snake_case_		    =     [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowercase						)       ,	__lowercase						)






						def        snake_case__						(						self							:     Any       ,	*__lowercase							:     Union[str, Any]       ,	**__lowercase							:     Tuple						):






							"""simple docstring"""






							pass






						def        snake_case__						(						self							:     int       ,	__lowercase							:     str=15						):






							"""simple docstring"""






							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"						):
									snake_case_		    =     self.rust_tokenizer_class.from_pretrained(__lowercase       ,	**__lowercase						)

									# Simple input
									snake_case_		    =     "This is a simple input"
									snake_case_		    =     ["This is a simple input 1", "This is a simple input 2"]
									snake_case_		    =     ("This is a simple input", "This is a pair")
									snake_case_		    =     [
									    ("This is a simple input 1", "This is a simple input 2"),
									    ("This is a simple pair 1", "This is a simple pair 2"),
									]

									# Simple input tests
									self.assertRaises(__lowercase       ,	tokenizer_r.encode       ,	__lowercase       ,	max_length=__lowercase       ,	padding="max_length"						)

									# Simple input
									self.assertRaises(__lowercase       ,	tokenizer_r.encode_plus       ,	__lowercase       ,	max_length=__lowercase       ,	padding="max_length"						)

									# Simple input
									self.assertRaises(
									    __lowercase       ,	tokenizer_r.batch_encode_plus       ,	__lowercase       ,	max_length=__lowercase       ,	padding="max_length"       ,	)

									# Pair input
									self.assertRaises(__lowercase       ,	tokenizer_r.encode       ,	__lowercase       ,	max_length=__lowercase       ,	padding="max_length"						)

									# Pair input
									self.assertRaises(__lowercase       ,	tokenizer_r.encode_plus       ,	__lowercase       ,	max_length=__lowercase       ,	padding="max_length"						)

									# Pair input
									self.assertRaises(
									    __lowercase       ,	tokenizer_r.batch_encode_plus       ,	__lowercase       ,	max_length=__lowercase       ,	padding="max_length"       ,	)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     CodeGenTokenizer.from_pretrained(self.tmpdirname       ,	pad_token="<pad>"						)

							# Simple input
							snake_case_		    =     "This is a simple input"
							snake_case_		    =     ["This is a simple input looooooooong", "This is a simple input"]
							snake_case_		    =     ("This is a simple input", "This is a pair")
							snake_case_		    =     [
							    ("This is a simple input loooooong", "This is a simple input"),
							    ("This is a simple pair loooooong", "This is a simple pair"),
							]

							snake_case_		    =     tokenizer.pad_token_id

							snake_case_		    =     tokenizer(__lowercase       ,	padding="max_length"       ,	max_length=30       ,	return_tensors="np"						)
							snake_case_		    =     tokenizer(__lowercase       ,	padding=__lowercase       ,	truncate=__lowercase       ,	return_tensors="np"						)
							snake_case_		    =     tokenizer(*__lowercase       ,	padding="max_length"       ,	max_length=60       ,	return_tensors="np"						)
							snake_case_		    =     tokenizer(__lowercase       ,	padding=__lowercase       ,	truncate=__lowercase       ,	return_tensors="np"						)

							# s
							# test single string max_length padding
							self.assertEqual(out_s["input_ids"].shape[-1]       ,	30						)
							self.assertTrue(pad_token_id in out_s["input_ids"]						)
							self.assertTrue(0 in out_s["attention_mask"]						)

							# s2
							# test automatic padding
							self.assertEqual(out_sa["input_ids"].shape[-1]       ,	33						)
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa["input_ids"][0]						)
							self.assertFalse(0 in out_sa["attention_mask"][0]						)
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa["input_ids"][1]						)
							self.assertTrue(0 in out_sa["attention_mask"][1]						)

							# p
							# test single pair max_length padding
							self.assertEqual(out_p["input_ids"].shape[-1]       ,	60						)
							self.assertTrue(pad_token_id in out_p["input_ids"]						)
							self.assertTrue(0 in out_p["attention_mask"]						)

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa["input_ids"].shape[-1]       ,	52						)
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa["input_ids"][0]						)
							self.assertFalse(0 in out_pa["attention_mask"][0]						)
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa["input_ids"][1]						)
							self.assertTrue(0 in out_pa["attention_mask"][1]						)






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     "$$$"
							snake_case_		    =     CodeGenTokenizer.from_pretrained(self.tmpdirname       ,	bos_token=__lowercase       ,	add_bos_token=__lowercase						)

							snake_case_		    =     "This is a simple input"
							snake_case_		    =     ["This is a simple input 1", "This is a simple input 2"]

							snake_case_		    =     tokenizer.bos_token_id

							snake_case_		    =     tokenizer(__lowercase						)
							snake_case_		    =     tokenizer(__lowercase						)

							self.assertEqual(out_s.input_ids[0]       ,	__lowercase						)
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

							snake_case_		    =     tokenizer.decode(out_s.input_ids						)
							snake_case_		    =     tokenizer.batch_decode(out_sa.input_ids						)

							self.assertEqual(decode_s.split()[0]       ,	__lowercase						)
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)






						@slow
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"						)

							snake_case_		    =     "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
							snake_case_		    =     "\nif len_a > len_b:      result = a\nelse:      result = b"

							snake_case_		    =     tokenizer.encode(__lowercase						)
							snake_case_		    =     ["^#", re.escape("<|endoftext|>"						), "^'''", "^\"\"\"", "\n\n\n"]
							snake_case_		    =     tokenizer.decode(__lowercase       ,	truncate_before_pattern=__lowercase						)
							self.assertEqual(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							pass
