


from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    AutoTokenizer,
						    TFAutoModelForCausalLM,
						    TFAutoModelForSeqaSeqLM,
						    TFAutoModelForSpeechSeqaSeq,
						    TFAutoModelForVisionaSeq,
						    TFBartForConditionalGeneration,
						    TFLogitsProcessorList,
						    TFMinLengthLogitsProcessor,
						    tf_top_k_top_p_filtering,
						)

if is_tensorflow_text_available():
						import tensorflow_text as text







@require_tf
class        snake_case_   (unittest.TestCase      ):
					def  lowerCamelCase__(      self   :str      )					->    Any:
												a__             =     tf.convert_to_tensor(
												    [
												        [
												            8.2_22_09_91,  # 3rd highest value; idx. 0
												            -0.5_62_00_44,
												            5.23_22_97_52,
												            4.0_38_63_93,
												            -6.8_79_83_78,
												            -0.54_78_58_02,
												            -3.2_01_21_53,
												            2.92_77_71_76,
												            1.88_17_19_53,
												            7.35_34_12_76,  # 5th highest value; idx. 9
												            8.43_20_78_33,  # 2nd highest value; idx. 10
												            -9.85_71_18_36,
												            -5.96_20_92_36,
												            -1.13_03_91_61,
												            -7.1_11_52_94,
												            -0.8_36_96_33,
												            -5.3_18_64_08,
												            7.06_42_74_07,
												            0.81_36_93_44,
												            -0.82_02_38_17,
												            -5.9_17_97_96,
												            0.58_81_34_43,
												            -6.99_77_84_38,
												            4.71_55_11_89,
												            -0.18_77_16_37,
												            7.44_02_07_59,  # 4th highest value; idx. 25
												            9.38_45_09_87,  # 1st highest value; idx. 26
												            2.12_66_29_41,
												            -9.32_56_20_38,
												            2.35_65_25_22,
												        ],  # cummulative prob of 5 highest values <= 0.6
												        [
												            0.58_42_55_18,
												            4.53_13_92_38,
												            -5.57_51_04_64,
												            -6.28_03_06_99,
												            -7.19_52_95_03,
												            -4.02_12_25_51,
												            1.39_33_70_37,
												            -6.06_70_70_57,
												            1.59_48_05_17,
												            -9.64_31_19,
												            0.03_90_77_99,
												            0.67_23_17_62,
												            -8.88_20_67_26,
												            6.27_11_59_22,  # 4th highest value; idx. 13
												            2.28_52_07_23,
												            4.82_76_75_06,
												            4.30_42_13_68,
												            8.8_27_53_13,  # 2nd highest value; idx. 17
												            5.44_02_99_58,  # 5th highest value; idx. 18
												            -4.4_73_57_94,
												            7.38_57_95_36,  # 3rd highest value; idx. 20
												            -2.91_05_16_63,
												            2.61_94_60_77,
												            -2.5_67_47_62,
												            -9.48_95_93_02,
												            -4.02_92_26_45,
												            -1.35_41_69_18,
												            9.67_70_23_23,  # 1st highest value; idx. 27
												            -5.89_47_85_53,
												            1.85_37_04_67,
												        ],  # cummulative prob of 5 highest values <= 0.6
												    ]			,dtype=tf.floataa			,)

												a__             =     tf.convert_to_tensor(
												    [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]			,dtype=tf.intaa			,)  # expected non filtered idx as noted above

												a__             =     tf.convert_to_tensor(
												    [8.22_20_99, 7.3_53_41_26, 8.43_20_78, 7.4_40_20_75, 9.3_84_51, 6.27_11_59, 8.82_75_31, 5.4_40_29_95, 7.3_85_79_56, 9.67_70_23]			,dtype=tf.floataa			,)  # expected non filtered values as noted above

												a__             =     tf_top_k_top_p_filtering(UpperCAmelCase_			,top_k=10			,top_p=0.6			,min_tokens_to_keep=4      )

												a__             =     output[output != -float('inf'      )]
												a__             =     tf.cast(
												    tf.where(tf.not_equal(UpperCAmelCase_			,tf.constant(-float('inf'      )			,dtype=tf.floataa      )      )      )			,dtype=tf.intaa			,)

												tf.debugging.assert_near(UpperCAmelCase_			,UpperCAmelCase_			,rtol=1E-12      )
												tf.debugging.assert_equal(UpperCAmelCase_			,UpperCAmelCase_      )







@require_tf
class        snake_case_   (unittest.TestCase	,						_UpperCAmelCase      ):
					# setting framework_dependent_parameters needs to be gated, just like its contents' imports
					if is_tf_available():
										UpperCAmelCase__					:    Any          =			{
										    """AutoModelForCausalLM""": TFAutoModelForCausalLM,
										    """AutoModelForSpeechSeq2Seq""": TFAutoModelForSpeechSeqaSeq,
										    """AutoModelForSeq2SeqLM""": TFAutoModelForSeqaSeqLM,
										    """AutoModelForVision2Seq""": TFAutoModelForVisionaSeq,
										    """LogitsProcessorList""": TFLogitsProcessorList,
										    """MinLengthLogitsProcessor""": TFMinLengthLogitsProcessor,
										    """create_tensor_fn""": tf.convert_to_tensor,
										    """floats_tensor""": floats_tensor,
										    """return_tensors""": """tf""",
										}
					@slow
					def  lowerCamelCase__(      self   :List[str]      )					->    Any:
												# TF-only test: tf.saved_model export
												a__             =     TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
												a__             =     2
												a__             =     2







												class        snake_case_   (tf.Module      ):
																	def __init__(      self   :Dict			,__snake_case   :Tuple      )					->    List[Any]:
																								super(UpperCAmelCase_			,self      ).__init__()
																								a__             =     model



																	@tf.function(
																	    input_signature=(
																	        tf.TensorSpec((None, input_length)			,tf.intaa			,name='input_ids'      ),
																	        tf.TensorSpec((None, input_length)			,tf.intaa			,name='attention_mask'      ),
																	    )			,jit_compile=UpperCAmelCase_			,)
																	def  lowerCamelCase__(      self   :Optional[int]			,__snake_case   :Tuple			,__snake_case   :Optional[Any]      )					->    List[str]:
																								a__             =     self.model.generate(
																								    input_ids=UpperCAmelCase_			,attention_mask=UpperCAmelCase_			,max_new_tokens=UpperCAmelCase_			,return_dict_in_generate=UpperCAmelCase_			,)
																								return {"sequences": outputs["sequences"]}





												a__             =     [[2, 0], [1_02, 1_03]]
												a__             =     [[1, 0], [1, 1]]
												a__             =     DummyModel(model=UpperCAmelCase_      )
												with tempfile.TemporaryDirectory() as tmp_dir:
																			tf.saved_model.save(UpperCAmelCase_			,UpperCAmelCase_			,signatures={'serving_default': dummy_model.serving}      )
																			a__             =     tf.saved_model.load(UpperCAmelCase_      ).signatures['serving_default']
																			for batch_size in range(1			,len(UpperCAmelCase_      ) + 1      ):
																										a__             =     {
																										    'input_ids': tf.constant(dummy_input_ids[:batch_size]      ),
																										    'attention_mask': tf.constant(dummy_attention_masks[:batch_size]      ),
																										}
																										a__             =     serving_func(**UpperCAmelCase_      )['sequences']
																										a__             =     test_model.generate(**UpperCAmelCase_			,max_new_tokens=UpperCAmelCase_      )
																										tf.debugging.assert_equal(UpperCAmelCase_			,UpperCAmelCase_      )
					@slow
					def  lowerCamelCase__(      self   :List[Any]      )					->    Union[str, Any]:
												# TF-only test: tf.saved_model export
												a__             =     TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
												a__             =     1
												a__             =     2







												class        snake_case_   (tf.Module      ):
																	def __init__(      self   :Tuple			,__snake_case   :str      )					->    Optional[Any]:
																								super(UpperCAmelCase_			,self      ).__init__()
																								a__             =     model



																	@tf.function(
																	    input_signature=(
																	        tf.TensorSpec((batch_size, None)			,tf.intaa			,name='input_ids'      ),
																	        tf.TensorSpec((batch_size, None)			,tf.intaa			,name='attention_mask'      ),
																	    )			,jit_compile=UpperCAmelCase_			,)
																	def  lowerCamelCase__(      self   :List[Any]			,__snake_case   :Any			,__snake_case   :List[Any]      )					->    Any:
																								a__             =     self.model.generate(
																								    input_ids=UpperCAmelCase_			,attention_mask=UpperCAmelCase_			,max_new_tokens=UpperCAmelCase_			,return_dict_in_generate=UpperCAmelCase_			,)
																								return {"sequences": outputs["sequences"]}





												a__             =     [[2], [1_02, 1_03]]
												a__             =     [[1], [1, 1]]
												a__             =     DummyModel(model=UpperCAmelCase_      )
												with tempfile.TemporaryDirectory() as tmp_dir:
																			tf.saved_model.save(UpperCAmelCase_			,UpperCAmelCase_			,signatures={'serving_default': dummy_model.serving}      )
																			a__             =     tf.saved_model.load(UpperCAmelCase_      ).signatures['serving_default']
																			for input_row in range(len(UpperCAmelCase_      )      ):
																										a__             =     {
																										    'input_ids': tf.constant([dummy_input_ids[input_row]]      ),
																										    'attention_mask': tf.constant([dummy_attention_masks[input_row]]      ),
																										}
																										a__             =     serving_func(**UpperCAmelCase_      )['sequences']
																										a__             =     test_model.generate(**UpperCAmelCase_			,max_new_tokens=UpperCAmelCase_      )
																										tf.debugging.assert_equal(UpperCAmelCase_			,UpperCAmelCase_      )
					@slow
					@require_tensorflow_text
					def  lowerCamelCase__(      self   :Any      )					->    List[Any]:
												# TF-only test: tf.saved_model export
												with tempfile.TemporaryDirectory() as tmp_dir:
																			# file needed to load the TF tokenizer
																			hf_hub_download(repo_id='google/flan-t5-small'			,filename='spiece.model'			,local_dir=UpperCAmelCase_      )







																			class        snake_case_   (tf.keras.layers.Layer      ):
																								def __init__(      self   :Tuple      )					->    Union[str, Any]:
																															super().__init__()
																															a__             =     text.SentencepieceTokenizer(
																															    model=tf.io.gfile.GFile(os.path.join(UpperCAmelCase_			,'spiece.model'      )			,'rb'      ).read()      )
																															a__             =     TFAutoModelForSeqaSeqLM.from_pretrained('hf-internal-testing/tiny-random-t5'      )



																								def  lowerCamelCase__(      self   :int			,__snake_case   :Tuple			,*__snake_case   :Tuple			,**__snake_case   :List[str]      )					->    Optional[int]:
																															a__             =     self.tokenizer.tokenize(UpperCAmelCase_      )
																															a__      ,					a__             =     text.pad_model_inputs(
																															    UpperCAmelCase_			,max_seq_length=64			,pad_value=self.model.config.pad_token_id      )
																															a__             =     self.model.generate(input_ids=UpperCAmelCase_			,attention_mask=UpperCAmelCase_      )
																															return self.tokenizer.detokenize(UpperCAmelCase_      )





																			a__             =     CompleteSentenceTransformer()
																			a__             =     tf.keras.layers.Input(shape=(1,)			,dtype=tf.string			,name='inputs'      )
																			a__             =     complete_model(UpperCAmelCase_      )
																			a__             =     tf.keras.Model(UpperCAmelCase_			,UpperCAmelCase_      )
																			keras_model.save(UpperCAmelCase_      )
					def  lowerCamelCase__(      self   :Optional[int]      )					->    List[str]:
												# Has PT equivalent: this test relies on random sampling
												a__             =     {
												    'do_sample': True,
												    'num_beams': 1,
												    'top_p': 0.7,
												    'top_k': 10,
												    'temperature': 0.7,
												}
												a__             =     14

												a__             =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
												a__             =     'Hello, my dog is cute and'
												a__             =     tokenizer(UpperCAmelCase_			,return_tensors='tf'      )
												a__             =     TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )

												a__             =     6_38
												# forces the generation to happen on CPU, to avoid GPU-related quirks
												with tf.device(':/CPU:0'      ):
																			tf.random.set_seed(0      )
																			a__             =     model.generate(**UpperCAmelCase_			,eos_token_id=UpperCAmelCase_			,**UpperCAmelCase_      )
												self.assertTrue(expectation == len(generated_tokens[0]      )      )

												a__             =     [6_38, 1_98]
												with tf.device(':/CPU:0'      ):
																			tf.random.set_seed(0      )
																			a__             =     model.generate(**UpperCAmelCase_			,eos_token_id=UpperCAmelCase_			,**UpperCAmelCase_      )
												self.assertTrue(expectation == len(generated_tokens[0]      )      )



					def  lowerCamelCase__(      self   :Dict      )					->    List[Any]:
												# Has PT equivalent: ample use of framework-specific code
												a__             =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bart'      )
												a__             =     'Hugging Face is a technology company based in New York and Paris.'
												a__             =     bart_tokenizer(UpperCAmelCase_			,return_tensors='tf'      ).input_ids
												a__             =     TFBartForConditionalGeneration.from_pretrained('hf-internal-testing/tiny-random-bart'      )
												a__             =     bart_model.generate(UpperCAmelCase_      ).numpy()







												class        snake_case_   (_UpperCAmelCase      ):
																	def  lowerCamelCase__(      self   :List[str]			,__snake_case   :Dict			,__snake_case   :Any=None			,**__snake_case   :List[Any]      )					->    Tuple:
																								return super().call(UpperCAmelCase_			,**UpperCAmelCase_      )





												a__             =     FakeBart.from_pretrained('hf-internal-testing/tiny-random-bart'      )
												a__             =     bart_model.generate(UpperCAmelCase_			,foo='bar'      ).numpy()
												self.assertTrue(np.array_equal(UpperCAmelCase_			,UpperCAmelCase_      )      )







												class        snake_case_   (bart_model.model.encoder.__class__      ):
																	def  lowerCamelCase__(      self   :Any			,__snake_case   :Any			,**__snake_case   :List[Any]      )					->    Tuple:
																								return super().call(UpperCAmelCase_			,**UpperCAmelCase_      )





												a__             =     FakeEncoder(bart_model.config			,bart_model.model.shared      )
												a__             =     fake_encoder

												# Normal generation still works (the output will be different because the encoder weights are different)
												a__             =     bart_model.generate(UpperCAmelCase_      ).numpy()
												with self.assertRaises(UpperCAmelCase_      ):
																			# FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
																			bart_model.generate(UpperCAmelCase_			,foo='bar'      )






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer




class 						lowercase__  (						_UpperCAmelCase  ):
	A__    :   Union[str, Any]				       ="""Wav2Vec2FeatureExtractor"""
	A__    :   Any				       ="""AutoTokenizer"""
	def __init__(			self							:   List[Any]       ,   UpperCAmelCase_							:   List[str]       ,   UpperCAmelCase_							:   str							):
						super().__init__(UpperCAmelCase_       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= self.feature_extractor
						SCREAMING_SNAKE_CASE__													= False
	@classmethod
	def      A_				(			cls							:   Union[str, Any]       ,   UpperCAmelCase_							:   Any       ,   **UpperCAmelCase_							:   Optional[Any]							):
						try:
											return super().from_pretrained(UpperCAmelCase_       ,   **UpperCAmelCase_							)
						except OSError:
											warnings.warn(
											    F'Loading a tokenizer inside {cls.__name__} from a config that does not'
											    ' include a `tokenizer_class` attribute is deprecated and will be '
											    'removed in v5. Please add `\'tokenizer_class\': \'Wav2Vec2CTCTokenizer\'`'
											    ' attribute to either your `config.json` or `tokenizer_config.json` '
											    'file to suppress this warning: '       ,   UpperCAmelCase_       ,   )

											SCREAMING_SNAKE_CASE__													= WavaVecaFeatureExtractor.from_pretrained(UpperCAmelCase_       ,   **UpperCAmelCase_							)
											SCREAMING_SNAKE_CASE__													= WavaVecaCTCTokenizer.from_pretrained(UpperCAmelCase_       ,   **UpperCAmelCase_							)

											return cls(feature_extractor=UpperCAmelCase_       ,   tokenizer=UpperCAmelCase_							)
	def __call__(			self							:   Any       ,   *UpperCAmelCase_							:   Dict       ,   **UpperCAmelCase_							:   Any							):
						# For backward compatibility
						if self._in_target_context_manager:
											return self.current_processor(*UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if "raw_speech" in kwargs:
											warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'							)
											SCREAMING_SNAKE_CASE__													= kwargs.pop('raw_speech'							)
						else:
											SCREAMING_SNAKE_CASE__													= kwargs.pop('audio'       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= kwargs.pop('sampling_rate'       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= kwargs.pop('text'       ,   UpperCAmelCase_							)
						if len(UpperCAmelCase_							) > 0:
											SCREAMING_SNAKE_CASE__													= args[0]
											SCREAMING_SNAKE_CASE__													= args[1:]

						if audio is None and text is None:
											raise ValueError('You need to specify either an `audio` or `text` input to process.'							)

						if audio is not None:
											SCREAMING_SNAKE_CASE__													= self.feature_extractor(UpperCAmelCase_       ,   *UpperCAmelCase_       ,   sampling_rate=UpperCAmelCase_       ,   **UpperCAmelCase_							)
						if text is not None:
											SCREAMING_SNAKE_CASE__													= self.tokenizer(UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if text is None:
											return inputs
						elif audio is None:
											return encodings
						else:
											SCREAMING_SNAKE_CASE__													= encodings['input_ids']
											return inputs
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   List[Any]       ,   **UpperCAmelCase_							:   Optional[int]							):
						# For backward compatibility
						if self._in_target_context_manager:
											return self.current_processor.pad(*UpperCAmelCase_       ,   **UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= kwargs.pop('input_features'       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= kwargs.pop('labels'       ,   UpperCAmelCase_							)
						if len(UpperCAmelCase_							) > 0:
											SCREAMING_SNAKE_CASE__													= args[0]
											SCREAMING_SNAKE_CASE__													= args[1:]

						if input_features is not None:
											SCREAMING_SNAKE_CASE__													= self.feature_extractor.pad(UpperCAmelCase_       ,   *UpperCAmelCase_       ,   **UpperCAmelCase_							)
						if labels is not None:
											SCREAMING_SNAKE_CASE__													= self.tokenizer.pad(UpperCAmelCase_       ,   **UpperCAmelCase_							)

						if labels is None:
											return input_features
						elif input_features is None:
											return labels
						else:
											SCREAMING_SNAKE_CASE__													= labels['input_ids']
											return input_features
	def      A_				(			self							:   List[Any]       ,   *UpperCAmelCase_							:   List[str]       ,   **UpperCAmelCase_							:   List[str]							):
						return self.tokenizer.batch_decode(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   Tuple       ,   *UpperCAmelCase_							:   int       ,   **UpperCAmelCase_							:   List[str]							):
						return self.tokenizer.decode(*UpperCAmelCase_       ,   **UpperCAmelCase_							)





	@contextmanager
	def      A_				(			self							:   str							):
						warnings.warn(
						    '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
						    'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
						    'your audio inputs, or in a separate call.'							)
						SCREAMING_SNAKE_CASE__													= True
						SCREAMING_SNAKE_CASE__													= self.tokenizer
						yield
						SCREAMING_SNAKE_CASE__													= self.feature_extractor
						SCREAMING_SNAKE_CASE__													= False





