
"""simple docstring"""





import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
    from transformers import WhisperFeatureExtractor, WhisperProcessor


_a									=				50_358
_a									=				50_362


@require_torch
@require_torchaudio
@require_sentencepiece
class A_      (unittest.TestCase ):




  '''simple docstring'''







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   List[Any]     	=       "openai/whisper-small.en"
     UpperCAmelCase_   :   List[Any]     	=       tempfile.mkdtemp()







  def 			UpperCamelCase__   (						self							,  **lowercase_    ):

     """simple docstring"""
     return WhisperTokenizer.from_pretrained(self.checkpoint							,  **lowercase_    )







  def 			UpperCamelCase__   (						self							,  **lowercase_    ):

     """simple docstring"""
     return WhisperFeatureExtractor.from_pretrained(self.checkpoint							,  **lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     shutil.rmtree(self.tmpdirname    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   str     	=       self.get_tokenizer()
     UpperCAmelCase_   :   Tuple     	=       self.get_feature_extractor()

     UpperCAmelCase_   :   List[str]     	=       WhisperProcessor(tokenizer=lowercase_							,  feature_extractor=lowercase_    )

     processor.save_pretrained(self.tmpdirname    )
     UpperCAmelCase_   :   int     	=       WhisperProcessor.from_pretrained(self.tmpdirname    )

     self.assertEqual(processor.tokenizer.get_vocab()							,  tokenizer.get_vocab()    )
     self.assertIsInstance(processor.tokenizer							,  lowercase_    )

     self.assertEqual(processor.feature_extractor.to_json_string()							,  feature_extractor.to_json_string()    )
     self.assertIsInstance(processor.feature_extractor							,  lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Union[str, Any]     	=       WhisperProcessor(tokenizer=self.get_tokenizer()							,  feature_extractor=self.get_feature_extractor()    )
     processor.save_pretrained(self.tmpdirname    )

     UpperCAmelCase_   :   Optional[Any]     	=       self.get_tokenizer(bos_token="(BOS)"							,  eos_token="(EOS)"    )
     UpperCAmelCase_   :   Optional[int]     	=       self.get_feature_extractor(do_normalize=lowercase_							,  padding_value=1.0    )

     UpperCAmelCase_   :   List[str]     	=       WhisperProcessor.from_pretrained(
         self.tmpdirname							,  bos_token="(BOS)"							,  eos_token="(EOS)"							,  do_normalize=lowercase_							,  padding_value=1.0    )

     self.assertEqual(processor.tokenizer.get_vocab()							,  tokenizer_add_kwargs.get_vocab()    )
     self.assertIsInstance(processor.tokenizer							,  lowercase_    )

     self.assertEqual(processor.feature_extractor.to_json_string()							,  feature_extractor_add_kwargs.to_json_string()    )
     self.assertIsInstance(processor.feature_extractor							,  lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   str     	=       self.get_feature_extractor()
     UpperCAmelCase_   :   Any     	=       self.get_tokenizer()

     UpperCAmelCase_   :   Dict     	=       WhisperProcessor(tokenizer=lowercase_							,  feature_extractor=lowercase_    )

     UpperCAmelCase_   :   Dict     	=       floats_list((3, 1000)    )

     UpperCAmelCase_   :   str     	=       feature_extractor(lowercase_							,  return_tensors="np"    )
     UpperCAmelCase_   :   Optional[int]     	=       processor(lowercase_							,  return_tensors="np"    )

     for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()							,  input_processor[key].sum()							,  delta=1E-2    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Dict     	=       self.get_feature_extractor()
     UpperCAmelCase_   :   List[Any]     	=       self.get_tokenizer()

     UpperCAmelCase_   :   List[Any]     	=       WhisperProcessor(tokenizer=lowercase_							,  feature_extractor=lowercase_    )

     UpperCAmelCase_   :   List[str]     	=       "This is a test string"

     UpperCAmelCase_   :   List[str]     	=       processor(text=lowercase_    )

     UpperCAmelCase_   :   Any     	=       tokenizer(lowercase_    )

     for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]							,  encoded_processor[key]    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Any     	=       self.get_feature_extractor()
     UpperCAmelCase_   :   List[Any]     	=       self.get_tokenizer()

     UpperCAmelCase_   :   List[str]     	=       WhisperProcessor(tokenizer=lowercase_							,  feature_extractor=lowercase_    )

     UpperCAmelCase_   :   int     	=       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     UpperCAmelCase_   :   List[str]     	=       processor.batch_decode(lowercase_    )
     UpperCAmelCase_   :   Optional[int]     	=       tokenizer.batch_decode(lowercase_    )

     self.assertListEqual(lowercase_							,  lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[int]     	=       self.get_feature_extractor()
     UpperCAmelCase_   :   Optional[Any]     	=       self.get_tokenizer()

     UpperCAmelCase_   :   Union[str, Any]     	=       WhisperProcessor(tokenizer=lowercase_							,  feature_extractor=lowercase_    )

     self.assertListEqual(
         processor.model_input_names							,  feature_extractor.model_input_names							,  msg="`processor` and `feature_extractor` model input names do not match"							,  )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Union[str, Any]     	=       self.get_feature_extractor()
     UpperCAmelCase_   :   List[Any]     	=       self.get_tokenizer()

     UpperCAmelCase_   :   Tuple     	=       WhisperProcessor(tokenizer=lowercase_							,  feature_extractor=lowercase_    )
     UpperCAmelCase_   :   Union[str, Any]     	=       processor.get_decoder_prompt_ids(task="transcribe"							,  no_timestamps=lowercase_    )

     self.assertIsInstance(lowercase_							,  lowercase_    )
     for ids in forced_decoder_ids:
        self.assertIsInstance(lowercase_							,  (list, tuple)    )

     UpperCAmelCase_   :   Optional[int]     	=       [TRANSCRIBE, NOTIMESTAMPS]
     self.assertListEqual([ids[-1] for ids in forced_decoder_ids]							,  lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[int]     	=       WhisperProcessor(tokenizer=self.get_tokenizer()							,  feature_extractor=self.get_feature_extractor()    )
     UpperCAmelCase_   :   Tuple     	=       processor.get_prompt_ids("Mr. Quilter"    )
     UpperCAmelCase_   :   Optional[Any]     	=       processor.tokenizer.decode(lowercase_    )

     self.assertListEqual(prompt_ids.tolist()							,  [5_0360, 1770, 13, 2264, 346, 353]    )
     self.assertEqual(lowercase_							,  "<|startofprev|> Mr. Quilter"    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[Any]     	=       WhisperProcessor(tokenizer=self.get_tokenizer()							,  feature_extractor=self.get_feature_extractor()    )
     UpperCAmelCase_   :   Union[str, Any]     	=       processor.get_prompt_ids(""    )
     UpperCAmelCase_   :   List[str]     	=       processor.tokenizer.decode(lowercase_    )

     self.assertListEqual(prompt_ids.tolist()							,  [5_0360, 220]    )
     self.assertEqual(lowercase_							,  "<|startofprev|> "    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Dict     	=       WhisperProcessor(tokenizer=self.get_tokenizer()							,  feature_extractor=self.get_feature_extractor()    )

     def _test_prompt_error_raised_helper(lowercase_							,  lowercase_    ):
        with pytest.raises(lowercase_    ) as excinfo:
           processor.get_prompt_ids(lowercase_    )
        UpperCAmelCase_   :   Optional[int]     	=       F"""Encountered text in the prompt corresponding to disallowed special token: {special_token}."""
        self.assertEqual(lowercase_							,  str(excinfo.value    )    )

     _test_prompt_error_raised_helper("<|startofprev|> test"							,  "<|startofprev|>"    )
     _test_prompt_error_raised_helper("test <|notimestamps|>"							,  "<|notimestamps|>"    )
     _test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>"							,  "<|zh|>"    )








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
 a_ 		=  None


a_ 		=  logging.get_logger(__name__)

a_ 		=  {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

a_ 		=  {
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"
        ),
    },
}

a_ 		=  {
    "xlm-roberta-base": 512,
    "xlm-roberta-large": 512,
    "xlm-roberta-large-finetuned-conll02-dutch": 512,
    "xlm-roberta-large-finetuned-conll02-spanish": 512,
    "xlm-roberta-large-finetuned-conll03-english": 512,
    "xlm-roberta-large-finetuned-conll03-german": 512,
}

class 			_lowerCamelCase    (      _SCREAMING_SNAKE_CASE ):

      """simple docstring"""

      lowerCAmelCase__      :			List[Any]     =       VOCAB_FILES_NAMES
      lowerCAmelCase__      :			List[Any]     =       PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__      :			Optional[Any]     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__      :			Optional[int]     =       ["input_ids", "attention_mask"]
      lowerCAmelCase__      :			Union[str, Any]     =       XLMRobertaTokenizer







      def __init__(  self       :			str			,							snake_case       :			Union[str, Any]=None			,							snake_case       :			Any=None			,							snake_case       :			Union[str, Any]="<s>"			,							snake_case       :			Tuple="</s>"			,							snake_case       :			List[Any]="</s>"			,							snake_case       :			Any="<s>"			,							snake_case       :			str="<unk>"			,							snake_case       :			int="<pad>"			,							snake_case       :			Any="<mask>"			,							**snake_case       :			str			,							):
          # Mask token behave like a normal word, i.e. include the space before it
          __UpperCamelCase			=    AddedToken(snake_case			,							lstrip=snake_case			,							rstrip=snake_case					) if isinstance(snake_case			,							snake_case					) else mask_token

          super().__init__(
              snake_case			,							tokenizer_file=snake_case			,							bos_token=snake_case			,							eos_token=snake_case			,							sep_token=snake_case			,							cls_token=snake_case			,							unk_token=snake_case			,							pad_token=snake_case			,							mask_token=snake_case			,							**snake_case			,							)

          __UpperCamelCase			=    vocab_file
          __UpperCamelCase			=    False if not self.vocab_file else True







      def    snake_case						(  self       :			List[Any]			,							snake_case       :			List[int]			,							snake_case       :			Optional[List[int]] = None					):

          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          __UpperCamelCase			=    [self.cls_token_id]
          __UpperCamelCase			=    [self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep







      def    snake_case						(  self       :			Any			,							snake_case       :			List[int]			,							snake_case       :			Optional[List[int]] = None					):
          __UpperCamelCase			=    [self.sep_token_id]
          __UpperCamelCase			=    [self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







      def    snake_case						(  self       :			Dict			,							snake_case       :			str			,							snake_case       :			Optional[str] = None					):
          if not self.can_save_slow_tokenizer:
              raise ValueError(
                  '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                  '''tokenizer.'''					)

          if not os.path.isdir(snake_case					):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory."					)
              return
          __UpperCamelCase			=    os.path.join(
              snake_case			,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(snake_case					):
              copyfile(self.vocab_file			,							snake_case					)

          return (out_vocab_file,)






