




"""simple docstring"""



import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
      from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

      from .test_feature_extraction_speechta import floats_list


SCREAMING_SNAKE_CASE       = get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")




@require_torch
class 			UpperCAmelCase_						(		unittest.TestCase							):




    def 							__magic_name__							(					self		: Optional[Any]   )   ->							str:




     '''simple docstring'''

     A__ =       tempfile.mkdtemp()

     A__ =       SpeechTaTokenizer(snake_case_   )
     tokenizer.save_pretrained(self.tmpdirname   )

     A__ =       {
         "feature_size": 1,
         "padding_value": 0.0,
         "sampling_rate": 16_000,
         "do_normalize": False,
         "num_mel_bins": 80,
         "hop_length": 16,
         "win_length": 64,
         "win_function": "hann_window",
         "fmin": 80,
         "fmax": 7_600,
         "mel_floor": 1e-10,
         "reduction_factor": 2,
         "return_attention_mask": True,
     }

     A__ =       os.path.join(self.tmpdirname			,				snake_case_   )
     with open(self.feature_extraction_file			,				"w"			,				encoding="utf-8"   ) as fp:
      fp.write(json.dumps(snake_case_   ) + "\n"   )




    def 							__magic_name__							(					self		: List[str]			,				**snake_case_		: Dict   )   ->							List[Any]:




     '''simple docstring'''

     return SpeechTaTokenizer.from_pretrained(self.tmpdirname			,				**snake_case_   )




    def 							__magic_name__							(					self		: Any			,				**snake_case_		: Any   )   ->							Dict:




     '''simple docstring'''

     return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname			,				**snake_case_   )




    def 							__magic_name__							(					self		: int   )   ->							Optional[Any]:




     '''simple docstring'''

     shutil.rmtree(self.tmpdirname   )




    def 							__magic_name__							(					self		: Dict   )   ->							List[str]:




     '''simple docstring'''

     A__ =       self.get_tokenizer()
     A__ =       self.get_feature_extractor()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     processor.save_pretrained(self.tmpdirname   )
     A__ =       SpeechTaProcessor.from_pretrained(self.tmpdirname   )

     self.assertEqual(processor.tokenizer.get_vocab()			,				tokenizer.get_vocab()   )
     self.assertIsInstance(processor.tokenizer			,				snake_case_   )

     self.assertEqual(processor.feature_extractor.to_json_string()			,				feature_extractor.to_json_string()   )
     self.assertIsInstance(processor.feature_extractor			,				snake_case_   )




    def 							__magic_name__							(					self		: Tuple   )   ->							Optional[int]:




     '''simple docstring'''

     A__ =       SpeechTaProcessor(tokenizer=self.get_tokenizer()			,				feature_extractor=self.get_feature_extractor()   )
     processor.save_pretrained(self.tmpdirname   )

     A__ =       self.get_tokenizer(bos_token="(BOS)"			,				eos_token="(EOS)"   )
     A__ =       self.get_feature_extractor(do_normalize=snake_case_			,				padding_value=1.0   )

     A__ =       SpeechTaProcessor.from_pretrained(
         self.tmpdirname			,				bos_token="(BOS)"			,				eos_token="(EOS)"			,				do_normalize=snake_case_			,				padding_value=1.0   )

     self.assertEqual(processor.tokenizer.get_vocab()			,				tokenizer_add_kwargs.get_vocab()   )
     self.assertIsInstance(processor.tokenizer			,				snake_case_   )

     self.assertEqual(processor.feature_extractor.to_json_string()			,				feature_extractor_add_kwargs.to_json_string()   )
     self.assertIsInstance(processor.feature_extractor			,				snake_case_   )




    def 							__magic_name__							(					self		: Tuple   )   ->							Union[str, Any]:




     '''simple docstring'''

     A__ =       self.get_feature_extractor()
     A__ =       self.get_tokenizer()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     A__ =       floats_list((3, 1_000)   )

     A__ =       feature_extractor(audio=snake_case_			,				return_tensors="np"   )
     A__ =       processor(audio=snake_case_			,				return_tensors="np"   )

     for key in input_feat_extract.keys():
      self.assertAlmostEqual(input_feat_extract[key].sum()			,				input_processor[key].sum()			,				delta=1e-2   )




    def 							__magic_name__							(					self		: Dict   )   ->							Union[str, Any]:




     '''simple docstring'''

     A__ =       self.get_feature_extractor()
     A__ =       self.get_tokenizer()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     A__ =       floats_list((3, 1_000)   )

     A__ =       feature_extractor(audio_target=snake_case_			,				return_tensors="np"   )
     A__ =       processor(audio_target=snake_case_			,				return_tensors="np"   )

     for key in input_feat_extract.keys():
      self.assertAlmostEqual(input_feat_extract[key].sum()			,				input_processor[key].sum()			,				delta=1e-2   )




    def 							__magic_name__							(					self		: Tuple   )   ->							List[str]:




     '''simple docstring'''

     A__ =       self.get_feature_extractor()
     A__ =       self.get_tokenizer()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     A__ =       "This is a test string"

     A__ =       processor(text=snake_case_   )
     A__ =       tokenizer(snake_case_   )

     for key in encoded_tok.keys():
      self.assertListEqual(encoded_tok[key]			,				encoded_processor[key]   )




    def 							__magic_name__							(					self		: List[str]   )   ->							str:




     '''simple docstring'''

     A__ =       self.get_feature_extractor()
     A__ =       self.get_tokenizer()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     A__ =       "This is a test string"

     A__ =       processor(text_target=snake_case_   )
     A__ =       tokenizer(snake_case_   )

     for key in encoded_tok.keys():
      self.assertListEqual(encoded_tok[key]			,				encoded_processor[key]   )




    def 							__magic_name__							(					self		: Dict   )   ->							Tuple:




     '''simple docstring'''

     A__ =       self.get_feature_extractor()
     A__ =       self.get_tokenizer()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     A__ =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     A__ =       processor.batch_decode(snake_case_   )
     A__ =       tokenizer.batch_decode(snake_case_   )

     self.assertListEqual(snake_case_			,				snake_case_   )




    def 							__magic_name__							(					self		: Tuple   )   ->							Union[str, Any]:




     '''simple docstring'''

     A__ =       self.get_feature_extractor()
     A__ =       self.get_tokenizer()

     A__ =       SpeechTaProcessor(tokenizer=snake_case_			,				feature_extractor=snake_case_   )

     self.assertListEqual(
         processor.model_input_names			,				feature_extractor.model_input_names			,				msg="`processor` and `feature_extractor` model input names do not match"			,				)











"""simple docstring"""



from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class 			UpperCAmelCase_						(		A_							):
    lowercase__		      =   '''MarkupLMFeatureExtractor'''
    lowercase__		      =   ('''MarkupLMTokenizer''', '''MarkupLMTokenizerFast''')
    lowercase__		      =   True




    def __call__(					self		: Dict			,				snake_case_		: Optional[int]=None			,				snake_case_		: Union[str, Any]=None			,				snake_case_		: Tuple=None			,				snake_case_		: str=None			,				snake_case_		: Optional[Any]=None			,				snake_case_		: bool = True			,				snake_case_		: Union[bool, str, PaddingStrategy] = False			,				snake_case_		: Union[bool, str, TruncationStrategy] = None			,				snake_case_		: Optional[int] = None			,				snake_case_		: int = 0			,				snake_case_		: Optional[int] = None			,				snake_case_		: Optional[bool] = None			,				snake_case_		: Optional[bool] = None			,				snake_case_		: bool = False			,				snake_case_		: bool = False			,				snake_case_		: bool = False			,				snake_case_		: bool = False			,				snake_case_		: bool = True			,				snake_case_		: Optional[Union[str, TensorType]] = None			,				**snake_case_		: Dict			,				)   ->							BatchEncoding:




     '''simple docstring'''

     if self.parse_html:
      if html_strings is None:
       raise ValueError("Make sure to pass HTML strings in case `parse_html` is set to `True`"   )

      if nodes is not None or xpaths is not None or node_labels is not None:
       raise ValueError(
           "Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"   )

      A__ =       self.feature_extractor(snake_case_   )
      A__ =       features["nodes"]
      A__ =       features["xpaths"]
     else:
      if html_strings is not None:
       raise ValueError("You have passed HTML strings but `parse_html` is set to `False`."   )
      if nodes is None or xpaths is None:
       raise ValueError("Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"   )

        # # second, apply the tokenizer
     if questions is not None and self.parse_html:
      if isinstance(snake_case_			,				snake_case_   ):
       A__ =       [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

     A__ =       self.tokenizer(
         text=questions if questions is not None else nodes			,				text_pair=nodes if questions is not None else None			,				xpaths=snake_case_			,				node_labels=snake_case_			,				add_special_tokens=snake_case_			,				padding=snake_case_			,				truncation=snake_case_			,				max_length=snake_case_			,				stride=snake_case_			,				pad_to_multiple_of=snake_case_			,				return_token_type_ids=snake_case_			,				return_attention_mask=snake_case_			,				return_overflowing_tokens=snake_case_			,				return_special_tokens_mask=snake_case_			,				return_offsets_mapping=snake_case_			,				return_length=snake_case_			,				verbose=snake_case_			,				return_tensors=snake_case_			,				**snake_case_			,				)

     return encoded_inputs




    def 							__magic_name__							(					self		: Optional[Any]			,				*snake_case_		: Any			,				**snake_case_		: Any   )   ->							int:




     '''simple docstring'''

     return self.tokenizer.batch_decode(*snake_case_			,				**snake_case_   )




    def 							__magic_name__							(					self		: Tuple			,				*snake_case_		: Optional[Any]			,				**snake_case_		: List[Any]   )   ->							Any:




     '''simple docstring'''

     return self.tokenizer.decode(*snake_case_			,				**snake_case_   )




    @property
    def 							__magic_name__							(					self		: List[Any]   )   ->							Optional[int]:




     '''simple docstring'''

     A__ =       self.tokenizer.model_input_names
     return tokenizer_input_names





