




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__       :      Union[str, Any]  =   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       :      int  =   {'vocab_file': 'spm_char.model'}

SCREAMING_SNAKE_CASE__       :      int  =   {
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

SCREAMING_SNAKE_CASE__       :      Union[str, Any]  =   {
    'microsoft/speecht5_asr': 1024,
    'microsoft/speecht5_tts': 1024,
    'microsoft/speecht5_vc': 1024,
}




class 		UpperCamelCase__	(lowerCAmelCase__     ):

    '''simple docstring'''



    lowerCamelCase_      :	Union[str, Any]	     =      VOCAB_FILES_NAMES
    lowerCamelCase_      :	Optional[int]	     =      PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase_      :	str	     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase_      :	List[Any]	     =      ["""input_ids""", """attention_mask"""]







    def __init__(    self     ,  UpperCamelCase__     ,  UpperCamelCase__="<s>"     ,  UpperCamelCase__="</s>"     ,  UpperCamelCase__="<unk>"     ,  UpperCamelCase__="<pad>"     ,  UpperCamelCase__ = None     ,  **UpperCamelCase__     ,  )     ->      None:
           lowerCamelCase   :      Optional[Any]			 =					{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               bos_token=UpperCamelCase__     ,  eos_token=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  pad_token=UpperCamelCase__     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase__     ,  )

           lowerCamelCase   :      str			 =					vocab_file

           lowerCamelCase   :      Any			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(UpperCamelCase__     )







    @property
    def     _lowercase     (    self     )     ->      Tuple:
           return self.sp_model.get_piece_size()







    def     _lowercase     (    self     )     ->      List[str]:
           lowerCamelCase   :      List[str]			 =					{self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
           vocab.update(self.added_tokens_encoder     )
           return vocab







    def __getstate__(    self     )     ->      int:
           lowerCamelCase   :      List[str]			 =					self.__dict__.copy()
           lowerCamelCase   :      Optional[int]			 =					None
           return state







    def __setstate__(    self     ,  UpperCamelCase__     )     ->      Optional[Any]:
           lowerCamelCase   :      Optional[Any]			 =					d

           # for backward compatibility
           if not hasattr(self     ,  "sp_model_kwargs"     ):
                  lowerCamelCase   :      Union[str, Any]			 =					{}

           lowerCamelCase   :      List[Any]			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(self.vocab_file     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      List[str]:
           return self.sp_model.encode(UpperCamelCase__     ,  out_type=UpperCamelCase__     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Tuple:
           return self.sp_model.piece_to_id(UpperCamelCase__     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Any:
           lowerCamelCase   :      Dict			 =					self.sp_model.IdToPiece(UpperCamelCase__     )
           return token







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Union[str, Any]:
           lowerCamelCase   :      List[Any]			 =					[]
           lowerCamelCase   :      Optional[Any]			 =					""
           for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                         out_string += self.sp_model.decode(UpperCamelCase__     ) + token
                         lowerCamelCase   :      int			 =					[]
                  else:
                         current_sub_tokens.append(UpperCamelCase__     )
           out_string += self.sp_model.decode(UpperCamelCase__     )
           return out_string.strip()







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__=None     )     ->      List[int]:
           if token_ids_a is None:
                  return token_ids_a + [self.eos_token_id]
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return token_ids_a + token_ids_a + [self.eos_token_id]







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     ,  UpperCamelCase__ = False     )     ->      List[int]:
           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=UpperCamelCase__     ,  token_ids_a=UpperCamelCase__     ,  already_has_special_tokens=UpperCamelCase__     )

           lowerCamelCase   :      Optional[int]			 =					[1]
           if token_ids_a is None:
                  return ([0] * len(UpperCamelCase__     )) + suffix_ones
           return ([0] * len(UpperCamelCase__     )) + ([0] * len(UpperCamelCase__     )) + suffix_ones







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      Tuple[str]:
           if not os.path.isdir(UpperCamelCase__     ):
                  logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
                  return
           lowerCamelCase   :      List[str]			 =					os.path.join(
               UpperCamelCase__     ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

           if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
                  copyfile(self.vocab_file     ,  UpperCamelCase__     )
           elif not os.path.isfile(self.vocab_file     ):
                  with open(UpperCamelCase__     ,  "wb"     ) as fi:
                         lowerCamelCase   :      Optional[int]			 =					self.sp_model.serialized_model_proto()
                         fi.write(UpperCamelCase__     )

           return (out_vocab_file,)







"""simple docstring"""




import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
				from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
				from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
				from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


__lowerCamelCase							 =     DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

__lowerCamelCase							 =     "main"
# Default branch name
__lowerCamelCase							 =     "f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
__lowerCamelCase							 =     "aaaaaaa"
# This commit does not exist, so we should 404.

__lowerCamelCase							 =     "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
__lowerCamelCase							 =     "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"






@contextlib.contextmanager
def      UpperCAmelCase    (		):






							"""simple docstring"""





							print('Welcome!' )
							yield
							print('Bye!' )






@contextlib.contextmanager
def      UpperCAmelCase    (		):






							"""simple docstring"""





							print('Bonjour!' )
							yield
							print('Au revoir!' )
class 			UpperCamelCase__(	unittest.TestCase       ):





						def  snake_case__		(				self   )    -> List[str]:
													# If the spec is missing, importlib would not be able to import the module dynamically.
													assert transformers.__spec__ is not None
													assert importlib.util.find_spec('transformers'   ) is not None






class 			UpperCamelCase__(	unittest.TestCase       ):





						@unittest.mock.patch('sys.stdout'						,new_callable=io.StringIO   )
						def  snake_case__		(				self						,__UpperCAmelCase   )    -> Dict:
													with ContextManagers([]   ):
																				print('Transformers are awesome!'   )
													# The print statement adds a new line at the end of the output
													self.assertEqual(mock_stdout.getvalue()						,'Transformers are awesome!\n'   )





						@unittest.mock.patch('sys.stdout'						,new_callable=io.StringIO   )
						def  snake_case__		(				self						,__UpperCAmelCase   )    -> List[str]:
													with ContextManagers([context_en()]   ):
																				print('Transformers are awesome!'   )
													# The output should be wrapped with an English welcome and goodbye
													self.assertEqual(mock_stdout.getvalue()						,'Welcome!\nTransformers are awesome!\nBye!\n'   )





						@unittest.mock.patch('sys.stdout'						,new_callable=io.StringIO   )
						def  snake_case__		(				self						,__UpperCAmelCase   )    -> Any:
													with ContextManagers([context_fr(), context_en()]   ):
																				print('Transformers are awesome!'   )
													# The output should be wrapped with an English and French welcome and goodbye
													self.assertEqual(mock_stdout.getvalue()						,'Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'   )





						@require_torch
						def  snake_case__		(				self   )    -> Union[str, Any]:
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels', 'next_sentence_label']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['start_positions', 'end_positions']   )
													class 			UpperCamelCase__(	__A       ):
																			pass






													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )





						@require_tf
						def  snake_case__		(				self   )    -> str:
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels', 'next_sentence_label']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['start_positions', 'end_positions']   )
													class 			UpperCamelCase__(	__A       ):
																			pass






													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )





						@require_flax
						def  snake_case__		(				self   )    -> List[Any]:
													# Flax models don't have labels
													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )
													class 			UpperCamelCase__(	__A       ):
																			pass






													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )





