code: 
  
  
  
  
  """simple docstring"""
  
  
  
  import shutil
  import tempfile
  import unittest
  
  import pytest
  
  from transformers import WhisperTokenizer, is_speech_available
  from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio
  
  from .test_feature_extraction_whisper import floats_list
  
  
  if is_speech_available():
  						from transformers import WhisperFeatureExtractor, WhisperProcessor
  
  
  SCREAMING_SNAKE_CASE       = 5_0358
  SCREAMING_SNAKE_CASE       = 5_0362
  
  
  
  
  @require_torch
  @require_torchaudio
  @require_sentencepiece
  class 			UpperCAmelCase_						(		unittest.TestCase							):
  
  
  
  
  				def 							__magic_name__							(					self		: int   )   ->							Optional[int]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       "openai/whisper-small.en"
  					A__ =       tempfile.mkdtemp()
  
  
  
  
  				def 							__magic_name__							(					self		: Tuple			,				**snake_case_		: Optional[Any]   )   ->							int:
  
  
  
  
  					'''simple docstring'''
  
  					return WhisperTokenizer.from_pretrained(self.checkpoint			,				**__SCREAMING_SNAKE_CASE   )
  
  
  
  
  				def 							__magic_name__							(					self		: Optional[int]			,				**snake_case_		: Dict   )   ->							Tuple:
  
  
  
  
  					'''simple docstring'''
  
  					return WhisperFeatureExtractor.from_pretrained(self.checkpoint			,				**__SCREAMING_SNAKE_CASE   )
  
  
  
  
  				def 							__magic_name__							(					self		: Dict   )   ->							str:
  
  
  
  
  					'''simple docstring'''
  
  					shutil.rmtree(self.tmpdirname   )
  
  
  
  
  				def 							__magic_name__							(					self		: List[Any]   )   ->							Union[str, Any]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       self.get_tokenizer()
  					A__ =       self.get_feature_extractor()
  
  					A__ =       WhisperProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,				feature_extractor=__SCREAMING_SNAKE_CASE   )
  
  					processor.save_pretrained(self.tmpdirname   )
  					A__ =       WhisperProcessor.from_pretrained(self.tmpdirname   )
  
  					self.assertEqual(processor.tokenizer.get_vocab()			,				tokenizer.get_vocab()   )
  					self.assertIsInstance(processor.tokenizer			,				__SCREAMING_SNAKE_CASE   )
  
  					self.assertEqual(processor.feature_extractor.to_json_string()			,				feature_extractor.to_json_string()   )
  					self.assertIsInstance(processor.feature_extractor			,				__SCREAMING_SNAKE_CASE   )
  
  
  
  
  				def 							__magic_name__							(					self		: Optional[int]   )   ->							Optional[Any]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       WhisperProcessor(tokenizer=self.get_tokenizer()			,				feature_extractor=self.get_feature_extractor()   )
  					processor.save_pretrained(self.tmpdirname   )
  
  					A__ =       self.get_tokenizer(bos_token="(BOS)"			,				eos_token="(EOS)"   )
  					A__ =       self.get_feature_extractor(do_normalize=__SCREAMING_SNAKE_CASE			,				padding_value=1.0   )
  
  					A__ =       WhisperProcessor.from_pretrained(
  					    self.tmpdirname			,				bos_token="(BOS)"			,				eos_token="(EOS)"			,				do_normalize=__SCREAMING_SNAKE_CASE			,				padding_value=1.0   )
  
  					self.assertEqual(processor.tokenizer.get_vocab()			,				tokenizer_add_kwargs.get_vocab()   )
  					self.assertIsInstance(processor.tokenizer			,				__SCREAMING_SNAKE_CASE   )
  
  					self.assertEqual(processor.feature_extractor.to_json_string()			,				feature_extractor_add_kwargs.to_json_string()   )
  					self.assertIsInstance(processor.feature_extractor			,				__SCREAMING_SNAKE_CASE   )
  
  
  
  
  				def 							__magic_name__							(					self		: Any   )   ->							str:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       self.get_feature_extractor()
  					A__ =       self.get_tokenizer()
  
  					A__ =       WhisperProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,				feature_extractor=__SCREAMING_SNAKE_CASE   )
  
  					A__ =       floats_list((3, 1_000)   )
  
  					A__ =       feature_extractor(__SCREAMING_SNAKE_CASE			,				return_tensors="np"   )
  					A__ =       processor(__SCREAMING_SNAKE_CASE			,				return_tensors="np"   )
  
  					for key in input_feat_extract.keys():
  						self.assertAlmostEqual(input_feat_extract[key].sum()			,				input_processor[key].sum()			,				delta=1e-2   )
  
  
  
  
  				def 							__magic_name__							(					self		: List[str]   )   ->							List[str]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       self.get_feature_extractor()
  					A__ =       self.get_tokenizer()
  
  					A__ =       WhisperProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,				feature_extractor=__SCREAMING_SNAKE_CASE   )
  
  					A__ =       "This is a test string"
  
  					A__ =       processor(text=__SCREAMING_SNAKE_CASE   )
  
  					A__ =       tokenizer(__SCREAMING_SNAKE_CASE   )
  
  					for key in encoded_tok.keys():
  						self.assertListEqual(encoded_tok[key]			,				encoded_processor[key]   )
  
  
  
  
  				def 							__magic_name__							(					self		: Union[str, Any]   )   ->							List[Any]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       self.get_feature_extractor()
  					A__ =       self.get_tokenizer()
  
  					A__ =       WhisperProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,				feature_extractor=__SCREAMING_SNAKE_CASE   )
  
  					A__ =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
  					A__ =       processor.batch_decode(__SCREAMING_SNAKE_CASE   )
  					A__ =       tokenizer.batch_decode(__SCREAMING_SNAKE_CASE   )
  
  					self.assertListEqual(__SCREAMING_SNAKE_CASE			,				__SCREAMING_SNAKE_CASE   )
  
  
  
  
  				def 							__magic_name__							(					self		: str   )   ->							List[str]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       self.get_feature_extractor()
  					A__ =       self.get_tokenizer()
  
  					A__ =       WhisperProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,				feature_extractor=__SCREAMING_SNAKE_CASE   )
  
  					self.assertListEqual(
  					    processor.model_input_names			,				feature_extractor.model_input_names			,				msg="`processor` and `feature_extractor` model input names do not match"			,				)
  
  
  
  
  				def 							__magic_name__							(					self		: Union[str, Any]   )   ->							int:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       self.get_feature_extractor()
  					A__ =       self.get_tokenizer()
  
  					A__ =       WhisperProcessor(tokenizer=__SCREAMING_SNAKE_CASE			,				feature_extractor=__SCREAMING_SNAKE_CASE   )
  					A__ =       processor.get_decoder_prompt_ids(task="transcribe"			,				no_timestamps=__SCREAMING_SNAKE_CASE   )
  
  					self.assertIsInstance(__SCREAMING_SNAKE_CASE			,				__SCREAMING_SNAKE_CASE   )
  					for ids in forced_decoder_ids:
  						self.assertIsInstance(__SCREAMING_SNAKE_CASE			,				(list, tuple)   )
  
  					A__ =       [TRANSCRIBE, NOTIMESTAMPS]
  					self.assertListEqual([ids[-1] for ids in forced_decoder_ids]			,				__SCREAMING_SNAKE_CASE   )
  
  
  
  
  				def 							__magic_name__							(					self		: List[str]   )   ->							Optional[Any]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       WhisperProcessor(tokenizer=self.get_tokenizer()			,				feature_extractor=self.get_feature_extractor()   )
  					A__ =       processor.get_prompt_ids("Mr. Quilter"   )
  					A__ =       processor.tokenizer.decode(__SCREAMING_SNAKE_CASE   )
  
  					self.assertListEqual(prompt_ids.tolist()			,				[50_360, 1_770, 13, 2_264, 346, 353]   )
  					self.assertEqual(__SCREAMING_SNAKE_CASE			,				"<|startofprev|> Mr. Quilter"   )
  
  
  
  
  				def 							__magic_name__							(					self		: List[str]   )   ->							str:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       WhisperProcessor(tokenizer=self.get_tokenizer()			,				feature_extractor=self.get_feature_extractor()   )
  					A__ =       processor.get_prompt_ids(""   )
  					A__ =       processor.tokenizer.decode(__SCREAMING_SNAKE_CASE   )
  
  					self.assertListEqual(prompt_ids.tolist()			,				[50_360, 220]   )
  					self.assertEqual(__SCREAMING_SNAKE_CASE			,				"<|startofprev|> "   )
  
  
  
  
  				def 							__magic_name__							(					self		: Any   )   ->							List[Any]:
  
  
  
  
  					'''simple docstring'''
  
  					A__ =       WhisperProcessor(tokenizer=self.get_tokenizer()			,				feature_extractor=self.get_feature_extractor()   )
  
  					def _test_prompt_error_raised_helper(snake_case_		: Dict			,				snake_case_		: List[str]   ):
  						with pytest.raises(__SCREAMING_SNAKE_CASE   ) as excinfo:
  							processor.get_prompt_ids(__SCREAMING_SNAKE_CASE   )
  						A__ =       F"""Encountered text in the prompt corresponding to disallowed special token: {special_token}."""
  						self.assertEqual(__SCREAMING_SNAKE_CASE			,				str(excinfo.value   )   )
  
  					_test_prompt_error_raised_helper("<|startofprev|> test"			,				"<|startofprev|>"   )
  					_test_prompt_error_raised_helper("test <|notimestamps|>"			,				"<|notimestamps|>"   )
  					_test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>"			,				"<|zh|>"   )
  
  
  
  
  
  
style_context: 
  
  
  
  
  """simple docstring"""
  
  
  
  from typing import Optional, Union
  
  from ...file_utils import TensorType
  from ...processing_utils import ProcessorMixin
  from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
  
  
  
  
  class 			UpperCAmelCase_						(		A_							):
      lowercase__		      =   '''MarkupLMFeatureExtractor'''
      lowercase__		      =   ('''MarkupLMTokenizer''', '''MarkupLMTokenizerFast''')
      lowercase__		      =   True
  
  
  
  
      def __call__(					self		: Dict			,				snake_case_		: Optional[int]=None			,				snake_case_		: Union[str, Any]=None			,				snake_case_		: Tuple=None			,				snake_case_		: str=None			,				snake_case_		: Optional[Any]=None			,				snake_case_		: bool = True			,				snake_case_		: Union[bool, str, PaddingStrategy] = False			,				snake_case_		: Union[bool, str, TruncationStrategy] = None			,				snake_case_		: Optional[int] = None			,				snake_case_		: int = 0			,				snake_case_		: Optional[int] = None			,				snake_case_		: Optional[bool] = None			,				snake_case_		: Optional[bool] = None			,				snake_case_		: bool = False			,				snake_case_		: bool = False			,				snake_case_		: bool = False			,				snake_case_		: bool = False			,				snake_case_		: bool = True			,				snake_case_		: Optional[Union[str, TensorType]] = None			,				**snake_case_		: Dict			,				)   ->							BatchEncoding:
  
  
  
  
       '''simple docstring'''
  
       if self.parse_html:
        if html_strings is None:
         raise ValueError("Make sure to pass HTML strings in case `parse_html` is set to `True`"   )
  
        if nodes is not None or xpaths is not None or node_labels is not None:
         raise ValueError(
             "Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"   )
  
        A__ =       self.feature_extractor(snake_case_   )
        A__ =       features["nodes"]
        A__ =       features["xpaths"]
       else:
        if html_strings is not None:
         raise ValueError("You have passed HTML strings but `parse_html` is set to `False`."   )
        if nodes is None or xpaths is None:
         raise ValueError("Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"   )
  
          # # second, apply the tokenizer
       if questions is not None and self.parse_html:
        if isinstance(snake_case_			,				snake_case_   ):
         A__ =       [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)
  
       A__ =       self.tokenizer(
           text=questions if questions is not None else nodes			,				text_pair=nodes if questions is not None else None			,				xpaths=snake_case_			,				node_labels=snake_case_			,				add_special_tokens=snake_case_			,				padding=snake_case_			,				truncation=snake_case_			,				max_length=snake_case_			,				stride=snake_case_			,				pad_to_multiple_of=snake_case_			,				return_token_type_ids=snake_case_			,				return_attention_mask=snake_case_			,				return_overflowing_tokens=snake_case_			,				return_special_tokens_mask=snake_case_			,				return_offsets_mapping=snake_case_			,				return_length=snake_case_			,				verbose=snake_case_			,				return_tensors=snake_case_			,				**snake_case_			,				)
  
       return encoded_inputs
  
  
  
  
      def 							__magic_name__							(					self		: Optional[Any]			,				*snake_case_		: Any			,				**snake_case_		: Any   )   ->							int:
  
  
  
  
       '''simple docstring'''
  
       return self.tokenizer.batch_decode(*snake_case_			,				**snake_case_   )
  
  
  
  
      def 							__magic_name__							(					self		: Tuple			,				*snake_case_		: Optional[Any]			,				**snake_case_		: List[Any]   )   ->							Any:
  
  
  
  
       '''simple docstring'''
  
       return self.tokenizer.decode(*snake_case_			,				**snake_case_   )
  
  
  
  
      @property
      def 							__magic_name__							(					self		: List[Any]   )   ->							Optional[int]:
  
  
  
  
       '''simple docstring'''
  
       A__ =       self.tokenizer.model_input_names
       return tokenizer_input_names
  
  
  
  
  
  