






"""simple docstring"""





import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
   from transformers import WhisperFeatureExtractor, WhisperProcessor


_a				   =							5_03_58
_a				   =							5_03_62




@require_torch
@require_torchaudio
@require_sentencepiece
class       _lowerCAmelCase						(			unittest.TestCase     ):







 """simple docstring"""







 def       _lowercase				(    self							:    Any			):
    __lowercase			=		"""openai/whisper-small.en"""
    __lowercase			=		tempfile.mkdtemp()


 def       _lowercase				(    self							:    List[str],      **UpperCAmelCase__							:    Tuple			):
    return WhisperTokenizer.from_pretrained(self.checkpoint,      **_lowerCAmelCase			)


 def       _lowercase				(    self							:    int,      **UpperCAmelCase__							:    List[Any]			):
    return WhisperFeatureExtractor.from_pretrained(self.checkpoint,      **_lowerCAmelCase			)


 def       _lowercase				(    self							:    str			):
    shutil.rmtree(self.tmpdirname			)


 def       _lowercase				(    self							:    Optional[Any]			):
    __lowercase			=		self.get_tokenizer()
    __lowercase			=		self.get_feature_extractor()

    __lowercase			=		WhisperProcessor(tokenizer=_lowerCAmelCase,      feature_extractor=_lowerCAmelCase			)

    processor.save_pretrained(self.tmpdirname			)
    __lowercase			=		WhisperProcessor.from_pretrained(self.tmpdirname			)

    self.assertEqual(processor.tokenizer.get_vocab(),      tokenizer.get_vocab()			)
    self.assertIsInstance(processor.tokenizer,      _lowerCAmelCase			)

    self.assertEqual(processor.feature_extractor.to_json_string(),      feature_extractor.to_json_string()			)
    self.assertIsInstance(processor.feature_extractor,      _lowerCAmelCase			)


 def       _lowercase				(    self							:    int			):
    __lowercase			=		WhisperProcessor(tokenizer=self.get_tokenizer(),      feature_extractor=self.get_feature_extractor()			)
    processor.save_pretrained(self.tmpdirname			)

    __lowercase			=		self.get_tokenizer(bos_token="(BOS)",      eos_token="(EOS)"			)
    __lowercase			=		self.get_feature_extractor(do_normalize=_lowerCAmelCase,      padding_value=1.0			)

    __lowercase			=		WhisperProcessor.from_pretrained(
        self.tmpdirname,      bos_token="(BOS)",      eos_token="(EOS)",      do_normalize=_lowerCAmelCase,      padding_value=1.0			)

    self.assertEqual(processor.tokenizer.get_vocab(),      tokenizer_add_kwargs.get_vocab()			)
    self.assertIsInstance(processor.tokenizer,      _lowerCAmelCase			)

    self.assertEqual(processor.feature_extractor.to_json_string(),      feature_extractor_add_kwargs.to_json_string()			)
    self.assertIsInstance(processor.feature_extractor,      _lowerCAmelCase			)


 def       _lowercase				(    self							:    Optional[int]			):
    __lowercase			=		self.get_feature_extractor()
    __lowercase			=		self.get_tokenizer()

    __lowercase			=		WhisperProcessor(tokenizer=_lowerCAmelCase,      feature_extractor=_lowerCAmelCase			)

    __lowercase			=		floats_list((3, 1_0_0_0)			)

    __lowercase			=		feature_extractor(_lowerCAmelCase,      return_tensors="np"			)
    __lowercase			=		processor(_lowerCAmelCase,      return_tensors="np"			)

    for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum(),      input_processor[key].sum(),      delta=1E-2			)


 def       _lowercase				(    self							:    int			):
    __lowercase			=		self.get_feature_extractor()
    __lowercase			=		self.get_tokenizer()

    __lowercase			=		WhisperProcessor(tokenizer=_lowerCAmelCase,      feature_extractor=_lowerCAmelCase			)

    __lowercase			=		"""This is a test string"""

    __lowercase			=		processor(text=_lowerCAmelCase			)

    __lowercase			=		tokenizer(_lowerCAmelCase			)

    for key in encoded_tok.keys():
       self.assertListEqual(encoded_tok[key],      encoded_processor[key]			)


 def       _lowercase				(    self							:    Dict			):
    __lowercase			=		self.get_feature_extractor()
    __lowercase			=		self.get_tokenizer()

    __lowercase			=		WhisperProcessor(tokenizer=_lowerCAmelCase,      feature_extractor=_lowerCAmelCase			)

    __lowercase			=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

    __lowercase			=		processor.batch_decode(_lowerCAmelCase			)
    __lowercase			=		tokenizer.batch_decode(_lowerCAmelCase			)

    self.assertListEqual(_lowerCAmelCase,      _lowerCAmelCase			)


 def       _lowercase				(    self							:    int			):
    __lowercase			=		self.get_feature_extractor()
    __lowercase			=		self.get_tokenizer()

    __lowercase			=		WhisperProcessor(tokenizer=_lowerCAmelCase,      feature_extractor=_lowerCAmelCase			)

    self.assertListEqual(
        processor.model_input_names,      feature_extractor.model_input_names,      msg="`processor` and `feature_extractor` model input names do not match",      )


 def       _lowercase				(    self							:    Any			):
    __lowercase			=		self.get_feature_extractor()
    __lowercase			=		self.get_tokenizer()

    __lowercase			=		WhisperProcessor(tokenizer=_lowerCAmelCase,      feature_extractor=_lowerCAmelCase			)
    __lowercase			=		processor.get_decoder_prompt_ids(task="transcribe",      no_timestamps=_lowerCAmelCase			)

    self.assertIsInstance(_lowerCAmelCase,      _lowerCAmelCase			)
    for ids in forced_decoder_ids:
       self.assertIsInstance(_lowerCAmelCase,      (list, tuple)			)

    __lowercase			=		[TRANSCRIBE, NOTIMESTAMPS]
    self.assertListEqual([ids[-1] for ids in forced_decoder_ids],      _lowerCAmelCase			)


 def       _lowercase				(    self							:    Tuple			):
    __lowercase			=		WhisperProcessor(tokenizer=self.get_tokenizer(),      feature_extractor=self.get_feature_extractor()			)
    __lowercase			=		processor.get_prompt_ids("Mr. Quilter"			)
    __lowercase			=		processor.tokenizer.decode(_lowerCAmelCase			)

    self.assertListEqual(prompt_ids.tolist(),      [5_0_3_6_0, 1_7_7_0, 1_3, 2_2_6_4, 3_4_6, 3_5_3]			)
    self.assertEqual(_lowerCAmelCase,      "<|startofprev|> Mr. Quilter"			)


 def       _lowercase				(    self							:    List[Any]			):
    __lowercase			=		WhisperProcessor(tokenizer=self.get_tokenizer(),      feature_extractor=self.get_feature_extractor()			)
    __lowercase			=		processor.get_prompt_ids(""			)
    __lowercase			=		processor.tokenizer.decode(_lowerCAmelCase			)

    self.assertListEqual(prompt_ids.tolist(),      [5_0_3_6_0, 2_2_0]			)
    self.assertEqual(_lowerCAmelCase,      "<|startofprev|> "			)



 def       _lowercase				(    self							:    List[Any]			):
    __lowercase			=		WhisperProcessor(tokenizer=self.get_tokenizer(),      feature_extractor=self.get_feature_extractor()			)

    def _test_prompt_error_raised_helper(UpperCAmelCase__							:    List[Any],      UpperCAmelCase__							:    List[Any]			):
       with pytest.raises(_lowerCAmelCase			) as excinfo:
          processor.get_prompt_ids(_lowerCAmelCase			)
       __lowercase			=		F"""Encountered text in the prompt corresponding to disallowed special token: {special_token}."""
       self.assertEqual(_lowerCAmelCase,      str(excinfo.value			)			)

    _test_prompt_error_raised_helper("<|startofprev|> test",      "<|startofprev|>"			)
    _test_prompt_error_raised_helper("test <|notimestamps|>",      "<|notimestamps|>"			)
    _test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>",      "<|zh|>"			)







"""simple docstring"""





import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Tuple:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Any:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_utf16_encoded.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w",    encoding="utf-16") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : Optional[int])			->		Optional[Any]:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts.json"
   __lowercase			=		textwrap.dedent(
       "\\n        [\n            {\"col_1\": -1},\n            {\"col_1\": 1, \"col_2\": 2},\n            {\"col_1\": 10, \"col_2\": 20}\n        ]\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		int:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts_field.json"
   __lowercase			=		textwrap.dedent(
       "\\n        {\n            \"field1\": 1,\n            \"field2\": \"aabb\",\n            \"field3\": [\n                {\"col_1\": -1},\n                {\"col_1\": 1, \"col_2\": 2},\n                {\"col_1\": 10, \"col_2\": 20}\n            ]\n        }\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        ("jsonl_file", {}),
        ("jsonl_file_utf16_encoded", {"encoding": "utf-16"}),
        ("json_file_with_list_of_dicts", {}),
        ("json_file_with_list_of_dicts_field", {"field": "field3"}),
    ],    )
def 						_A						(						UpperCamelCase_  : Any,    UpperCamelCase_  : List[Any],    UpperCamelCase_  : Optional[Any])			->		List[Any]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        (
            "jsonl_file",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts_field",
            {
                "field": "field3",
                "features": Features(
                    {"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")}),
            },
        ),
    ],    )
def 						_A						(						UpperCamelCase_  : Optional[int],    UpperCamelCase_  : int,    UpperCamelCase_  : str)			->		Optional[int]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}




