


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case				:    Any       				=     logging.get_logger(__name__)

snake_case				:    List[Any]       				=     {"vocab_file": "sentencepiece.bpe.model"}

snake_case				:    List[str]       				=     {
    "vocab_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
    }
}

snake_case				:    Any       				=     {
    "camembert-base": 5_12,
}

snake_case				:    List[Any]       				=     "▁"







class 		_snake_case  (      _snake_case							):
 SCREAMING_SNAKE_CASE__      =       VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE__      =       ['input_ids', 'attention_mask']




 def __init__(		self					,  _lowerCamelCase					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="<unk>"					,  _lowerCamelCase="<pad>"					,  _lowerCamelCase="<mask>"					,  _lowerCamelCase=["<s>NOTUSED", "</s>NOTUSED"]					,  _lowerCamelCase = None					,  **_lowerCamelCase					,  ):
       # Mask token behave like a normal word, i.e. include the space before it
       a						:Dict 							=       AddedToken(_lowerCAmelCase					,  lstrip=_lowerCAmelCase					,  rstrip=_lowerCAmelCase    ) if isinstance(_lowerCAmelCase					,  _lowerCAmelCase    ) else mask_token

       a						:Optional[int] 							=       {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=_lowerCAmelCase					,  eos_token=_lowerCAmelCase					,  unk_token=_lowerCAmelCase					,  sep_token=_lowerCAmelCase					,  cls_token=_lowerCAmelCase					,  pad_token=_lowerCAmelCase					,  mask_token=_lowerCAmelCase					,  additional_special_tokens=_lowerCAmelCase					,  sp_model_kwargs=self.sp_model_kwargs					,  **_lowerCAmelCase					,  )
       a						:Dict 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(str(_lowerCAmelCase    )    )
       a						:Tuple 							=       vocab_file
       # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
       # sentencepiece vocabulary (this is the case for <s> and </s>
       a						:int 							=       {"<s>NOTUSED": 0, "<pad>": 1, "</s>NOTUSED": 2, "<unk>": 3}
       a						:Tuple 							=       len(self.fairseq_tokens_to_ids    )
       a						:int 							=       len(self.sp_model    ) + len(self.fairseq_tokens_to_ids    )
       a						:Any 							=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):

       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       a						:List[Any] 							=       [self.cls_token_id]
       a						:List[str] 							=       [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None					,  _lowerCamelCase = False    ):
       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCAmelCase					,  token_ids_a=_lowerCAmelCase					,  already_has_special_tokens=_lowerCAmelCase    )

       if token_ids_a is None:
             return [1] + ([0] * len(_lowerCAmelCase    )) + [1]
       return [1] + ([0] * len(_lowerCAmelCase    )) + [1, 1] + ([0] * len(_lowerCAmelCase    )) + [1]




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       a						:Any 							=       [self.sep_token_id]
       a						:str 							=       [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       return len(self.fairseq_tokens_to_ids    ) + len(self.sp_model    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       {self.convert_ids_to_tokens(_lowerCAmelCase    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       return self.sp_model.encode(_lowerCAmelCase					,  out_type=_lowerCAmelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
       elif self.sp_model.PieceToId(_lowerCAmelCase    ) == 0:
             # Convert sentence piece unk token to fairseq unk token index
             return self.unk_token_id
       return self.fairseq_offset + self.sp_model.PieceToId(_lowerCAmelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       a						:Tuple 							=       []
       a						:Any 							=       ""
       a						:str 							=       False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   if not prev_is_special:
                         out_string += " "
                   out_string += self.sp_model.decode(_lowerCAmelCase    ) + token
                   a						:Optional[int] 							=       True
                   a						:Any 							=       []
             else:
                   current_sub_tokens.append(_lowerCAmelCase    )
                   a						:str 							=       False
       out_string += self.sp_model.decode(_lowerCAmelCase    )
       return out_string.strip()




 def __getstate__(		self    ):
       a						:int 							=       self.__dict__.copy()
       a						:List[str] 							=       None
       return state




 def __setstate__(		self					,  _lowerCamelCase    ):
       a						:int 							=       d

       # for backward compatibility
       if not hasattr(self					,  '''sp_model_kwargs'''    ):
             a						:Optional[int] 							=       {}

       a						:List[str] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(self.vocab_file    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       if not os.path.isdir(_lowerCAmelCase    ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
             return
       a						:Any 							=       os.path.join(
           _lowerCAmelCase					,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(_lowerCAmelCase    ) and os.path.isfile(self.vocab_file    ):
             copyfile(self.vocab_file					,  _lowerCAmelCase    )
       elif not os.path.isfile(self.vocab_file    ):
             with open(_lowerCAmelCase					,  '''wb'''    ) as fi:
                   a						:Dict 							=       self.sp_model.serialized_model_proto()
                   fi.write(_lowerCAmelCase    )

       return (out_vocab_file,)











from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
     from ...feature_extraction_utils import FeatureExtractionMixin
     from ...tokenization_utils_base import PreTrainedTokenizerBase
     from ...utils import TensorType

_UpperCAmelCase		:  Tuple	=   logging.get_logger(__name__)

_UpperCAmelCase		:  Dict	=   {
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}

# fmt: off
_UpperCAmelCase		:  Union[str, Any]	=   [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
_UpperCAmelCase		:  List[Any]	=   [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]




class       __lowerCAmelCase				(							lowerCAmelCase):
       _a           =  '''whisper'''
       _a           =  ['''past_key_values''']
       _a           =  {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}






       def __init__(      self:    int				,    _lowerCAmelCase:    str=5_18_65				,    _lowerCAmelCase:    str=80				,    _lowerCAmelCase:    int=6				,    _lowerCAmelCase:    Tuple=4				,    _lowerCAmelCase:    Union[str, Any]=6				,    _lowerCAmelCase:    List[Any]=4				,    _lowerCAmelCase:    Any=15_36				,    _lowerCAmelCase:    Union[str, Any]=15_36				,    _lowerCAmelCase:    str=0.0				,    _lowerCAmelCase:    str=0.0				,    _lowerCAmelCase:    List[Any]=5_02_57				,    _lowerCAmelCase:    Optional[Any]=True				,    _lowerCAmelCase:    Tuple=True				,    _lowerCAmelCase:    str="gelu"				,    _lowerCAmelCase:    Dict=2_56				,    _lowerCAmelCase:    Union[str, Any]=0.0				,    _lowerCAmelCase:    Any=0.0				,    _lowerCAmelCase:    Dict=0.0				,    _lowerCAmelCase:    Union[str, Any]=0.02				,    _lowerCAmelCase:    Any=False				,    _lowerCAmelCase:    List[str]=15_00				,    _lowerCAmelCase:    Tuple=4_48				,    _lowerCAmelCase:    Optional[Any]=5_02_56				,    _lowerCAmelCase:    Dict=5_02_56				,    _lowerCAmelCase:    List[Any]=5_02_56				,    _lowerCAmelCase:    Union[str, Any]=None				,    _lowerCAmelCase:    str=[2_20, 5_02_56]				,    _lowerCAmelCase:    Optional[int]=False				,    _lowerCAmelCase:    Optional[int]=2_56				,    _lowerCAmelCase:    int=False				,    _lowerCAmelCase:    Dict=0.05				,    _lowerCAmelCase:    Optional[Any]=10				,    _lowerCAmelCase:    List[str]=2				,    _lowerCAmelCase:    Tuple=0.0				,    _lowerCAmelCase:    str=10				,    _lowerCAmelCase:    Union[str, Any]=0				,    _lowerCAmelCase:    List[Any]=7				,    **_lowerCAmelCase:    Union[str, Any]				,    ):
         lowercase			:Optional[Any]							=  vocab_size
         lowercase			:Optional[int]							=  num_mel_bins
         lowercase			:Union[str, Any]							=  d_model
         lowercase			:List[Any]							=  encoder_layers
         lowercase			:Optional[Any]							=  encoder_attention_heads
         lowercase			:Union[str, Any]							=  decoder_layers
         lowercase			:List[str]							=  decoder_attention_heads
         lowercase			:Optional[int]							=  decoder_ffn_dim
         lowercase			:List[Any]							=  encoder_ffn_dim
         lowercase			:Optional[Any]							=  dropout
         lowercase			:Tuple							=  attention_dropout
         lowercase			:Tuple							=  activation_dropout
         lowercase			:Optional[Any]							=  activation_function
         lowercase			:Any							=  init_std
         lowercase			:Optional[int]							=  encoder_layerdrop
         lowercase			:Optional[int]							=  decoder_layerdrop
         lowercase			:str							=  use_cache
         lowercase			:Optional[Any]							=  encoder_layers
         lowercase			:List[Any]							=  scale_embedding  # scale factor will be sqrt(d_model) if True
         lowercase			:Any							=  max_source_positions
         lowercase			:Optional[Any]							=  max_target_positions

         # Audio Classification-specific parameters. Feel free to ignore for other classes.
         lowercase			:int							=  classifier_proj_size
         lowercase			:List[str]							=  use_weighted_layer_sum

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         lowercase			:Tuple							=  apply_spec_augment
         lowercase			:int							=  mask_time_prob
         lowercase			:Union[str, Any]							=  mask_time_length
         lowercase			:Dict							=  mask_time_min_masks
         lowercase			:Tuple							=  mask_feature_prob
         lowercase			:List[Any]							=  mask_feature_length
         lowercase			:List[Any]							=  mask_feature_min_masks

         lowercase			:Any							=  median_filter_width

         super().__init__(
             pad_token_id=_lowerCAmelCase				,    bos_token_id=_lowerCAmelCase				,    eos_token_id=_lowerCAmelCase				,    is_encoder_decoder=_lowerCAmelCase				,    decoder_start_token_id=_lowerCAmelCase				,    suppress_tokens=_lowerCAmelCase				,    begin_suppress_tokens=_lowerCAmelCase				,    **_lowerCAmelCase				,    )







class       __lowerCAmelCase				(							lowerCAmelCase):






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:Tuple							=  OrderedDict(
             [
                 ("input_features", {0: "batch", 1: "feature_size", 2: "encoder_sequence"}),
             ] )
         if self.use_past:
           lowercase			:List[Any]							=  {0: "batch"}
         else:
           lowercase			:str							=  {0: "batch", 1: "decoder_sequence"}

         if self.use_past:
           self.fill_with_past_key_values_(_lowerCAmelCase				,    direction="inputs" )

         return common_inputs






       def     SCREAMING_SNAKE_CASE			(      self:    Dict				,    _lowerCAmelCase:    Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]				,    _lowerCAmelCase:    int = -1				,    _lowerCAmelCase:    int = -1				,    _lowerCAmelCase:    bool = False				,    _lowerCAmelCase:    Optional["TensorType"] = None				,    _lowerCAmelCase:    int = 2_20_50				,    _lowerCAmelCase:    float = 5.0				,    _lowerCAmelCase:    int = 2_20				,    ):
         lowercase			:List[str]							=  OrderedDict()
         lowercase			:str							=  OnnxConfig.generate_dummy_inputs(
             self				,    preprocessor=preprocessor.feature_extractor				,    batch_size=_lowerCAmelCase				,    framework=_lowerCAmelCase				,    sampling_rate=_lowerCAmelCase				,    time_duration=_lowerCAmelCase				,    frequency=_lowerCAmelCase				,    )
         lowercase			:Optional[Any]							=  encoder_inputs["input_features"].shape[2]
         lowercase			:List[str]							=  encoder_sequence_length // 2 if self.use_past else seq_length

         lowercase			:Dict							=  super().generate_dummy_inputs(
             preprocessor.tokenizer				,    _lowerCAmelCase				,    _lowerCAmelCase				,    _lowerCAmelCase				,    _lowerCAmelCase )

         lowercase			:str							=  encoder_inputs.pop("input_features" )
         lowercase			:Optional[int]							=  decoder_inputs.pop("decoder_input_ids" )

         if "past_key_values" in decoder_inputs:
           lowercase			:List[str]							=  decoder_inputs.pop("past_key_values" )

         return dummy_inputs






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         return 1e-3



