



"""simple docstring"""






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
   lowerCAmelCase__        =				'''pt'''
elif is_tf_available():
   lowerCAmelCase__        =				'''tf'''
else:
   lowerCAmelCase__        =				'''jax'''






class        __snake_case       (	_lowercase  ,     unittest.TestCase):
  snake_case__				:   Union[str, Any]  			=						PerceiverTokenizer
  snake_case__				:   Union[str, Any]  			=						False




  def 						SCREAMING_SNAKE_CASE				( self     :						str		):






         """simple docstring"""






         super().setUp()
         _lowerCamelCase						:				Any    			= PerceiverTokenizer()
         tokenizer.save_pretrained(self.tmpdirname		)




  @cached_property
  def 						SCREAMING_SNAKE_CASE				( self     :						Tuple		):






         """simple docstring"""






         return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''		)




  def 						SCREAMING_SNAKE_CASE				( self     :						int						,					**__lowerCAmelCase     :						Optional[Any]		):






         """simple docstring"""






         return self.tokenizer_class.from_pretrained(self.tmpdirname						,					**__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Union[str, Any]						,					__lowerCAmelCase     :						int						,					__lowerCAmelCase     :						List[Any]=False						,					__lowerCAmelCase     :						Tuple=2_0						,					__lowerCAmelCase     :						List[str]=5		):






         """simple docstring"""






         _lowerCamelCase						:				Optional[int]    			= []
         for i in range(len(__lowerCAmelCase		)		):
                try:
                       _lowerCamelCase						:				Optional[Any]    			= tokenizer.decode([i]						,					clean_up_tokenization_spaces=__lowerCAmelCase		)
                except UnicodeDecodeError:
                       pass
                toks.append((i, tok)		)

         _lowerCamelCase						:				Any    			= list(filter(lambda __lowerCAmelCase		: re.match(R'''^[ a-zA-Z]+$'''						,					t[1]		)						,					__lowerCAmelCase		)		)
         _lowerCamelCase						:				Optional[Any]    			= list(filter(lambda __lowerCAmelCase		: [t[0]] == tokenizer.encode(t[1]						,					add_special_tokens=__lowerCAmelCase		)						,					__lowerCAmelCase		)		)
         if max_length is not None and len(__lowerCAmelCase		) > max_length:
                _lowerCamelCase						:				Optional[Any]    			= toks[:max_length]
         if min_length is not None and len(__lowerCAmelCase		) < min_length and len(__lowerCAmelCase		) > 0:
                while len(__lowerCAmelCase		) < min_length:
                       _lowerCamelCase						:				Tuple    			= toks + toks
        # toks_str = [t[1] for t in toks]
         _lowerCamelCase						:				List[Any]    			= [t[0] for t in toks]

         # Ensure consistency
         _lowerCamelCase						:				Any    			= tokenizer.decode(__lowerCAmelCase						,					clean_up_tokenization_spaces=__lowerCAmelCase		)
         if " " not in output_txt and len(__lowerCAmelCase		) > 1:
                _lowerCamelCase						:				int    			= (
                    tokenizer.decode([toks_ids[0]]						,					clean_up_tokenization_spaces=__lowerCAmelCase		)
                    + ''' '''
                    + tokenizer.decode(toks_ids[1:]						,					clean_up_tokenization_spaces=__lowerCAmelCase		)
                )
         if with_prefix_space:
                _lowerCamelCase						:				Any    			= ''' ''' + output_txt
         _lowerCamelCase						:				int    			= tokenizer.encode(__lowerCAmelCase						,					add_special_tokens=__lowerCAmelCase		)
         return output_txt, output_ids




  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:				Tuple    			= self.perceiver_tokenizer
         _lowerCamelCase						:				Optional[Any]    			= '''Unicode €.'''
         _lowerCamelCase						:				str    			= tokenizer(__lowerCAmelCase		)
         _lowerCamelCase						:				List[Any]    			= [4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
         self.assertEqual(encoded['''input_ids''']						,					__lowerCAmelCase		)

         # decoding
         _lowerCamelCase						:				Union[str, Any]    			= tokenizer.decode(__lowerCAmelCase		)
         self.assertEqual(__lowerCAmelCase						,					'''[CLS]Unicode €.[SEP]'''		)

         _lowerCamelCase						:				Union[str, Any]    			= tokenizer('''e è é ê ë'''		)
         _lowerCamelCase						:				int    			= [4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
         self.assertEqual(encoded['''input_ids''']						,					__lowerCAmelCase		)
         # decoding
         _lowerCamelCase						:				List[Any]    			= tokenizer.decode(__lowerCAmelCase		)
         self.assertEqual(__lowerCAmelCase						,					'''[CLS]e è é ê ë[SEP]'''		)

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''		)		)						,					'''[CLS]e è é ê ë[SEP]'''		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Dict		):






         """simple docstring"""






         _lowerCamelCase						:				List[str]    			= self.perceiver_tokenizer
         _lowerCamelCase						:				List[Any]    			= ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
         # fmt: off
         _lowerCamelCase						:				Dict    			= [4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
         # fmt: on
         _lowerCamelCase						:				Optional[Any]    			= tokenizer(__lowerCAmelCase						,					padding=__lowerCAmelCase						,					return_tensors=__lowerCAmelCase		)
         self.assertIsInstance(__lowerCAmelCase						,					__lowerCAmelCase		)

         if FRAMEWORK != "jax":
                _lowerCamelCase						:				str    			= list(batch.input_ids.numpy()[0]		)
         else:
                _lowerCamelCase						:				Any    			= list(batch.input_ids.tolist()[0]		)

         self.assertListEqual(__lowerCAmelCase						,					__lowerCAmelCase		)

         self.assertEqual((2, 3_8)						,					batch.input_ids.shape		)
         self.assertEqual((2, 3_8)						,					batch.attention_mask.shape		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Union[str, Any]		):






         """simple docstring"""






         _lowerCamelCase						:				List[Any]    			= self.perceiver_tokenizer
         _lowerCamelCase						:				Optional[int]    			= ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
         _lowerCamelCase						:				Union[str, Any]    			= tokenizer(__lowerCAmelCase						,					padding=__lowerCAmelCase						,					return_tensors=__lowerCAmelCase		)
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn('''input_ids'''						,					__lowerCAmelCase		)
         self.assertIn('''attention_mask'''						,					__lowerCAmelCase		)
         self.assertNotIn('''decoder_input_ids'''						,					__lowerCAmelCase		)
         self.assertNotIn('''decoder_attention_mask'''						,					__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				Optional[int]    			= self.perceiver_tokenizer
         _lowerCamelCase						:				str    			= [
             '''Summary of the text.''',
             '''Another summary.''',
         ]
         _lowerCamelCase						:				int    			= tokenizer(
             text_target=__lowerCAmelCase						,					max_length=3_2						,					padding='''max_length'''						,					truncation=__lowerCAmelCase						,					return_tensors=__lowerCAmelCase		)
         self.assertEqual(3_2						,					targets['''input_ids'''].shape[1]		)




  def 						SCREAMING_SNAKE_CASE				( self     :						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				int    			= self.get_tokenizers()
         for tokenizer in tokenizers:
                with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                       self.assertNotEqual(tokenizer.model_max_length						,					4_2		)

        # Now let's start the test
         _lowerCamelCase						:				List[str]    			= self.get_tokenizers()
         for tokenizer in tokenizers:
                with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                       # Isolate this from the other tests because we save additional tokens/etc
                       _lowerCamelCase						:				Optional[int]    			= tempfile.mkdtemp()

                       _lowerCamelCase						:				Optional[int]    			= ''' He is very happy, UNwant\u00E9d,running'''
                       _lowerCamelCase						:				Dict    			= tokenizer.encode(__lowerCAmelCase						,					add_special_tokens=__lowerCAmelCase		)
                       tokenizer.save_pretrained(__lowerCAmelCase		)

                       _lowerCamelCase						:				Any    			= tokenizer.__class__.from_pretrained(__lowerCAmelCase		)
                       _lowerCamelCase						:				List[str]    			= after_tokenizer.encode(__lowerCAmelCase						,					add_special_tokens=__lowerCAmelCase		)
                       self.assertListEqual(__lowerCAmelCase						,					__lowerCAmelCase		)

                       shutil.rmtree(__lowerCAmelCase		)

         _lowerCamelCase						:				str    			= self.get_tokenizers(model_max_length=4_2		)
         for tokenizer in tokenizers:
                with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                       # Isolate this from the other tests because we save additional tokens/etc
                       _lowerCamelCase						:				List[Any]    			= tempfile.mkdtemp()

                       _lowerCamelCase						:				Optional[Any]    			= ''' He is very happy, UNwant\u00E9d,running'''
                       tokenizer.add_tokens(['''bim''', '''bambam''']		)
                       _lowerCamelCase						:				Tuple    			= tokenizer.additional_special_tokens
                       additional_special_tokens.append('''new_additional_special_token'''		)
                       tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}		)
                       _lowerCamelCase						:				Any    			= tokenizer.encode(__lowerCAmelCase						,					add_special_tokens=__lowerCAmelCase		)
                       tokenizer.save_pretrained(__lowerCAmelCase		)

                       _lowerCamelCase						:				Optional[Any]    			= tokenizer.__class__.from_pretrained(__lowerCAmelCase		)
                       _lowerCamelCase						:				Dict    			= after_tokenizer.encode(__lowerCAmelCase						,					add_special_tokens=__lowerCAmelCase		)
                       self.assertListEqual(__lowerCAmelCase						,					__lowerCAmelCase		)
                       self.assertIn('''new_additional_special_token'''						,					after_tokenizer.additional_special_tokens		)
                       self.assertEqual(after_tokenizer.model_max_length						,					4_2		)

                       _lowerCamelCase						:				Tuple    			= tokenizer.__class__.from_pretrained(__lowerCAmelCase						,					model_max_length=4_3		)
                       self.assertEqual(tokenizer.model_max_length						,					4_3		)

                       shutil.rmtree(__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				List[str]    			= []
         if self.test_slow_tokenizer:
                tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

         if self.test_rust_tokenizer:
                tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

         for tokenizer_class, tokenizer_utils in tokenizer_list:
                with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer_utils.save_pretrained(__lowerCAmelCase		)

                       with open(os.path.join(__lowerCAmelCase						,					'''special_tokens_map.json'''		)						,					encoding='''utf-8'''		) as json_file:
                              _lowerCamelCase						:				Union[str, Any]    			= json.load(__lowerCAmelCase		)

                       with open(os.path.join(__lowerCAmelCase						,					'''tokenizer_config.json'''		)						,					encoding='''utf-8'''		) as json_file:
                              _lowerCamelCase						:				Optional[int]    			= json.load(__lowerCAmelCase		)

                       _lowerCamelCase						:				str    			= [f'''<extra_id_{i}>''' for i in range(1_2_5		)]

                       _lowerCamelCase						:				Optional[int]    			= added_tokens_extra_ids + [
                           '''an_additional_special_token'''
                       ]
                       _lowerCamelCase						:				Any    			= added_tokens_extra_ids + [
                           '''an_additional_special_token'''
                       ]

                       with open(os.path.join(__lowerCAmelCase						,					'''special_tokens_map.json'''		)						,					'''w'''						,					encoding='''utf-8'''		) as outfile:
                              json.dump(__lowerCAmelCase						,					__lowerCAmelCase		)
                       with open(os.path.join(__lowerCAmelCase						,					'''tokenizer_config.json'''		)						,					'''w'''						,					encoding='''utf-8'''		) as outfile:
                              json.dump(__lowerCAmelCase						,					__lowerCAmelCase		)

                       # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                       # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                       # "special_tokens_map.json" files
                       _lowerCamelCase						:				Dict    			= tokenizer_class.from_pretrained(
                           __lowerCAmelCase						,					)
                       self.assertIn(
                           '''an_additional_special_token'''						,					tokenizer_without_change_in_init.additional_special_tokens		)
                       self.assertEqual(
                           ['''an_additional_special_token''']						,					tokenizer_without_change_in_init.convert_ids_to_tokens(
                               tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']		)		)						,					)

                       # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                       _lowerCamelCase						:				Union[str, Any]    			= added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''						,					lstrip=__lowerCAmelCase		)]
                       _lowerCamelCase						:				int    			= tokenizer_class.from_pretrained(
                           __lowerCAmelCase						,					additional_special_tokens=__lowerCAmelCase						,					)

                       self.assertIn('''a_new_additional_special_token'''						,					tokenizer.additional_special_tokens		)
                       self.assertEqual(
                           ['''a_new_additional_special_token''']						,					tokenizer.convert_ids_to_tokens(
                               tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']		)		)						,					)




  def 						SCREAMING_SNAKE_CASE				( self     :						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				Optional[Any]    			= self.perceiver_tokenizer
         self.assertEqual(tokenizer.decode([1_7_8]		)						,					'''�'''		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Dict		):






         """simple docstring"""






         pass




  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[int]		):






         """simple docstring"""






         pass




  def 						SCREAMING_SNAKE_CASE				( self     :						str		):






         """simple docstring"""






         pass




  def 						SCREAMING_SNAKE_CASE				( self     :						Tuple		):






         """simple docstring"""






         pass




  def 						SCREAMING_SNAKE_CASE				( self     :						Any		):






         """simple docstring"""






         _lowerCamelCase						:				Optional[int]    			= self.get_tokenizers(fast=__lowerCAmelCase						,					do_lower_case=__lowerCAmelCase		)
         for tokenizer in tokenizers:
                with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                       _lowerCamelCase						:				Any    			= ['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
                       _lowerCamelCase						:				Dict    			= tokenizer.convert_tokens_to_string(__lowerCAmelCase		)

                       self.assertIsInstance(__lowerCAmelCase						,					__lowerCAmelCase		)









import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						"""Speech2TextFeatureExtractor"""
			lowerCAmelCase_							     =						"""Speech2TextTokenizer"""





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				super().__init__(__lowerCAmelCase       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.feature_extractor
				lowerCamelCase__								=		False





			def __call__(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				if self._in_target_context_manager:
					return self.current_processor(*__lowerCAmelCase       ,  **__lowerCAmelCase							)

				if "raw_speech" in kwargs:
					warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''							)
					lowerCamelCase__								=		kwargs.pop('''raw_speech'''							)
				else:
					lowerCamelCase__								=		kwargs.pop('''audio'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		kwargs.pop('''sampling_rate'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		kwargs.pop('''text'''       ,  __lowerCAmelCase							)
				if len(__lowerCAmelCase							) > 0:
					lowerCamelCase__								=		args[0]
					lowerCamelCase__								=		args[1:]

				if audio is None and text is None:
					raise ValueError('''You need to specify either an `audio` or `text` input to process.'''							)

				if audio is not None:
					lowerCamelCase__								=		self.feature_extractor(__lowerCAmelCase       ,  *__lowerCAmelCase       ,  sampling_rate=__lowerCAmelCase       ,  **__lowerCAmelCase							)
				if text is not None:
					lowerCamelCase__								=		self.tokenizer(__lowerCAmelCase       ,  **__lowerCAmelCase							)

				if text is None:
					return inputs
				elif audio is None:
					return encodings
				else:
					lowerCamelCase__								=		encodings['''input_ids''']
					return inputs





			def 		__lowerCamelCase				(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.batch_decode(*__lowerCAmelCase       ,  **__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.decode(*__lowerCAmelCase       ,  **__lowerCAmelCase							)





			@contextmanager
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				warnings.warn(
				    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
				    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
				    '''your audio inputs, or in a separate call.'''							)
				lowerCamelCase__								=		True
				lowerCamelCase__								=		self.tokenizer
				yield
				lowerCamelCase__								=		self.feature_extractor
				lowerCamelCase__								=		False



