code: 
  '''simple docstring'''
  
  import shutil
  import tempfile
  import unittest
  from pathlib import Path
  from shutil import copyfile
  
  from transformers import SpeechaTextTokenizer, is_speech_available
  from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
  from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
  from transformers.utils import FEATURE_EXTRACTOR_NAME
  
  from .test_feature_extraction_speech_to_text import floats_list
  
  
  if is_speech_available():
       from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor
  
  
  UpperCamelCase_     			=       get_tests_dir("""fixtures/test_sentencepiece.model""")
  
  
  @require_torch
  @require_torchaudio
  @require_sentencepiece
  class   a_  (unittest.TestCase    ):
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  List[Any]   				=					tempfile.mkdtemp()
  
           _lowerCAmelCase       :  Optional[Any]   				=					['<s>', '<pad>', '</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est']
           _lowerCAmelCase       :  Optional[Any]   				=					dict(zip(snake_case_   ,   range(len(snake_case_  )  )  )  )
           _lowerCAmelCase       :  List[str]   				=					Path(self.tmpdirname  )
           save_json(snake_case_   ,   save_dir / VOCAB_FILES_NAMES["""vocab_file"""]  )
           if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
             copyfile(snake_case_   ,   save_dir / VOCAB_FILES_NAMES["""spm_file"""]  )
  
           _lowerCAmelCase       :  Optional[int]   				=					SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )
           tokenizer.save_pretrained(self.tmpdirname  )
  
           _lowerCAmelCase       :  List[Any]   				=					{
               'feature_size': 2_4,
               'num_mel_bins': 2_4,
               'padding_value': 0.0,
               'sampling_rate': 1_6_0_0_0,
               'return_attention_mask': False,
               'do_normalize': True,
           }
           save_json(snake_case_   ,   save_dir / FEATURE_EXTRACTOR_NAME  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   **snake_case_  ):
           return SpeechaTextTokenizer.from_pretrained(self.tmpdirname   ,   **snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   **snake_case_  ):
           return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname   ,   **snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           shutil.rmtree(self.tmpdirname  )
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  Dict   				=					self.get_tokenizer()
           _lowerCAmelCase       :  Optional[Any]   				=					self.get_feature_extractor()
  
           _lowerCAmelCase       :  List[Any]   				=					SpeechaTextProcessor(tokenizer=snake_case_   ,   feature_extractor=snake_case_  )
  
           processor.save_pretrained(self.tmpdirname  )
           _lowerCAmelCase       :  Optional[int]   				=					SpeechaTextProcessor.from_pretrained(self.tmpdirname  )
  
           self.assertEqual(processor.tokenizer.get_vocab()   ,   tokenizer.get_vocab()  )
           self.assertIsInstance(processor.tokenizer   ,   snake_case_  )
  
           self.assertEqual(processor.feature_extractor.to_json_string()   ,   feature_extractor.to_json_string()  )
           self.assertIsInstance(processor.feature_extractor   ,   snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  List[str]   				=					SpeechaTextProcessor(
               tokenizer=self.get_tokenizer()   ,   feature_extractor=self.get_feature_extractor()  )
           processor.save_pretrained(self.tmpdirname  )
  
           _lowerCAmelCase       :  int   				=					self.get_tokenizer(bos_token="""(BOS)"""   ,   eos_token="""(EOS)"""  )
           _lowerCAmelCase       :  List[Any]   				=					self.get_feature_extractor(do_normalize=snake_case_   ,   padding_value=1.0  )
  
           _lowerCAmelCase       :  List[str]   				=					SpeechaTextProcessor.from_pretrained(
               self.tmpdirname   ,   bos_token="""(BOS)"""   ,   eos_token="""(EOS)"""   ,   do_normalize=snake_case_   ,   padding_value=1.0  )
  
           self.assertEqual(processor.tokenizer.get_vocab()   ,   tokenizer_add_kwargs.get_vocab()  )
           self.assertIsInstance(processor.tokenizer   ,   snake_case_  )
  
           self.assertEqual(processor.feature_extractor.to_json_string()   ,   feature_extractor_add_kwargs.to_json_string()  )
           self.assertIsInstance(processor.feature_extractor   ,   snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  Tuple   				=					self.get_feature_extractor()
           _lowerCAmelCase       :  Optional[int]   				=					self.get_tokenizer()
  
           _lowerCAmelCase       :  Optional[int]   				=					SpeechaTextProcessor(tokenizer=snake_case_   ,   feature_extractor=snake_case_  )
  
           _lowerCAmelCase       :  Any   				=					floats_list((3, 1_0_0_0)  )
  
           _lowerCAmelCase       :  Optional[int]   				=					feature_extractor(snake_case_   ,   return_tensors="""np"""  )
           _lowerCAmelCase       :  Union[str, Any]   				=					processor(snake_case_   ,   return_tensors="""np"""  )
  
           for key in input_feat_extract.keys():
             self.assertAlmostEqual(input_feat_extract[key].sum()   ,   input_processor[key].sum()   ,   delta=1E-2  )
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  Optional[int]   				=					self.get_feature_extractor()
           _lowerCAmelCase       :  Optional[int]   				=					self.get_tokenizer()
  
           _lowerCAmelCase       :  List[str]   				=					SpeechaTextProcessor(tokenizer=snake_case_   ,   feature_extractor=snake_case_  )
  
           _lowerCAmelCase       :  List[str]   				=					'This is a test string'
  
           _lowerCAmelCase       :  List[Any]   				=					processor(text=snake_case_  )
  
           _lowerCAmelCase       :  Union[str, Any]   				=					tokenizer(snake_case_  )
  
           for key in encoded_tok.keys():
             self.assertListEqual(encoded_tok[key]   ,   encoded_processor[key]  )
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  int   				=					self.get_feature_extractor()
           _lowerCAmelCase       :  str   				=					self.get_tokenizer()
  
           _lowerCAmelCase       :  int   				=					SpeechaTextProcessor(tokenizer=snake_case_   ,   feature_extractor=snake_case_  )
  
           _lowerCAmelCase       :  int   				=					[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
           _lowerCAmelCase       :  int   				=					processor.batch_decode(snake_case_  )
           _lowerCAmelCase       :  Union[str, Any]   				=					tokenizer.batch_decode(snake_case_  )
  
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
  
  
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  Optional[Any]   				=					self.get_feature_extractor()
           _lowerCAmelCase       :  int   				=					self.get_tokenizer()
  
           _lowerCAmelCase       :  int   				=					SpeechaTextProcessor(tokenizer=snake_case_   ,   feature_extractor=snake_case_  )
  
           self.assertListEqual(
               processor.model_input_names   ,   feature_extractor.model_input_names   ,   msg="""`processor` and `feature_extractor` model input names do not match"""   ,   )
  
  
  
style_context: 
  '''simple docstring'''
  
  import sys
  from collections.abc import Mapping
  
  import numpy as np
  import pyarrow as pa
  
  from .. import config
  from ..utils.py_utils import map_nested
  from .formatting import TensorFormatter
  
  
  class   a_  (TensorFormatter[Mapping, np.ndarray, Mapping]    ):
  
  
  
  
         def __init__(     self   ,   snake_case_=None   ,   **snake_case_  ):
           super().__init__(features=snake_case_  )
           _lowerCAmelCase       :  Any   				=					np_array_kwargs
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           if isinstance(snake_case_   ,   snake_case_  ):
             if column and all(
                 isinstance(snake_case_   ,   np.ndarray  ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column  ):
               return np.stack(snake_case_  )
             else:
               # don't use np.array(column, dtype=object)
               # since it fails in certain cases
               # see https://stackoverflow.com/q/51005699
               _lowerCAmelCase       :  str   				=					np.empty(len(snake_case_  )   ,   dtype=snake_case_  )
               _lowerCAmelCase       :  List[Any]   				=					column
               return out
           return column
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           if isinstance(snake_case_   ,   (str, bytes, type(snake_case_  ))  ):
             return value
           elif isinstance(snake_case_   ,   (np.character, np.ndarray)  ) and np.issubdtype(value.dtype   ,   np.character  ):
             return value
           elif isinstance(snake_case_   ,   np.number  ):
             return value
  
           _lowerCAmelCase       :  List[Any]   				=					{}
  
           if isinstance(snake_case_   ,   np.ndarray  ) and np.issubdtype(value.dtype   ,   np.integer  ):
             _lowerCAmelCase       :  Union[str, Any]   				=					{"""dtype""": np.intaa}
           elif isinstance(snake_case_   ,   np.ndarray  ) and np.issubdtype(value.dtype   ,   np.floating  ):
             _lowerCAmelCase       :  Tuple   				=					{"""dtype""": np.floataa}
           elif config.PIL_AVAILABLE and "PIL" in sys.modules:
             import PIL.Image
  
             if isinstance(snake_case_   ,   PIL.Image.Image  ):
               return np.asarray(snake_case_   ,   **self.np_array_kwargs  )
  
           return np.asarray(snake_case_   ,   **{**default_dtype, **self.np_array_kwargs}  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           # support for torch, tf, jax etc.
           if config.TORCH_AVAILABLE and "torch" in sys.modules:
             import torch
  
             if isinstance(snake_case_   ,   torch.Tensor  ):
               return self._tensorize(data_struct.detach().cpu().numpy()[()]  )
           if hasattr(snake_case_   ,   """__array__"""  ) and not isinstance(snake_case_   ,   (np.ndarray, np.character, np.number)  ):
             _lowerCAmelCase       :  List[Any]   				=					data_struct.__array__()
           # support for nested types like struct of list of struct
           if isinstance(snake_case_   ,   np.ndarray  ):
             if data_struct.dtype == object:
               return self._consolidate([self.recursive_tensorize(snake_case_  ) for substruct in data_struct]  )
           if isinstance(snake_case_   ,   (list, tuple)  ):
             return self._consolidate([self.recursive_tensorize(snake_case_  ) for substruct in data_struct]  )
           return self._tensorize(snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           return map_nested(self._recursive_tensorize   ,   snake_case_   ,   map_list=snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase       :  Optional[Any]   				=					self.numpy_arrow_extractor().extract_row(snake_case_  )
           _lowerCAmelCase       :  Any   				=					self.python_features_decoder.decode_row(snake_case_  )
           return self.recursive_tensorize(snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase       :  List[Any]   				=					self.numpy_arrow_extractor().extract_column(snake_case_  )
           _lowerCAmelCase       :  Tuple   				=					self.python_features_decoder.decode_column(snake_case_   ,   pa_table.column_names[0]  )
           _lowerCAmelCase       :  Optional[int]   				=					self.recursive_tensorize(snake_case_  )
           _lowerCAmelCase       :  Optional[Any]   				=					self._consolidate(snake_case_  )
           return column
  
  
  
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase       :  List[Any]   				=					self.numpy_arrow_extractor().extract_batch(snake_case_  )
           _lowerCAmelCase       :  str   				=					self.python_features_decoder.decode_batch(snake_case_  )
           _lowerCAmelCase       :  Dict   				=					self.recursive_tensorize(snake_case_  )
           for column_name in batch:
             _lowerCAmelCase       :  int   				=					self._consolidate(batch[column_name]  )
           return batch
  
  
  