


'''simple docstring'''


import json
import os
from typing import Optional

import numpy as np

from ...feature_extraction_utils import BatchFeature
from ...processing_utils import ProcessorMixin
from ...utils import logging
from ...utils.hub import get_file_from_repo
from ..auto import AutoTokenizer


lowerCamelCase__					=   logging.get_logger(__name__)






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		str				      =   "AutoTokenizer"
			lowerCAmelCase       :		Tuple				      =   ["tokenizer"]

			lowerCAmelCase       :		Dict				      =   {
			    "semantic_prompt": 1,
			    "coarse_prompt": 2,
			    "fine_prompt": 2,
			}




			def __init__(						self     :   Dict		,     lowerCamelCase__     :   Tuple		,     lowerCamelCase__     :   Union[str, Any]=None       )      ->Optional[Any]:



					'''simple docstring'''







					super().__init__(lowerCamelCase__       )

					_UpperCAmelCase						:				Dict	    =					speaker_embeddings




			@classmethod
			def 							lowerCAmelCase__			(						cls     :   Optional[Any]		,     lowerCamelCase__     :   Dict		,     lowerCamelCase__     :   Optional[int]="speaker_embeddings_path.json"		,     **lowerCamelCase__     :   int       )      ->Any:



					'''simple docstring'''







					if speaker_embeddings_dict_path is not None:
							_UpperCAmelCase						:				List[str]	    =					get_file_from_repo(
							    lowerCamelCase__		,     lowerCamelCase__		,     subfolder=kwargs.pop("subfolder"		,     lowerCamelCase__       )		,     cache_dir=kwargs.pop("cache_dir"		,     lowerCamelCase__       )		,     force_download=kwargs.pop("force_download"		,     lowerCamelCase__       )		,     proxies=kwargs.pop("proxies"		,     lowerCamelCase__       )		,     resume_download=kwargs.pop("resume_download"		,     lowerCamelCase__       )		,     local_files_only=kwargs.pop("local_files_only"		,     lowerCamelCase__       )		,     use_auth_token=kwargs.pop("use_auth_token"		,     lowerCamelCase__       )		,     revision=kwargs.pop("revision"		,     lowerCamelCase__       )		,     )
							if speaker_embeddings_path is None:
									logger.warning(
									    F"""`{os.path.join(lowerCamelCase__		,     lowerCamelCase__       )}` does not exists
                    , no preloaded speaker embeddings will be used - Make sure to provide a correct path to the json
                    dictionnary if wanted, otherwise set `speaker_embeddings_dict_path=None`."""       )
									_UpperCAmelCase						:				int	    =					None
							else:
									with open(lowerCamelCase__       ) as speaker_embeddings_json:
											_UpperCAmelCase						:				Optional[Any]	    =					json.load(lowerCamelCase__       )
					else:
							_UpperCAmelCase						:				Union[str, Any]	    =					None

					_UpperCAmelCase						:				List[Any]	    =					AutoTokenizer.from_pretrained(lowerCamelCase__		,     **lowerCamelCase__       )

					return cls(tokenizer=lowerCamelCase__		,     speaker_embeddings=lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   Optional[Any]		,     lowerCamelCase__     :   Optional[int]		,     lowerCamelCase__     :   Optional[Any]="speaker_embeddings_path.json"		,     lowerCamelCase__     :   Tuple="speaker_embeddings"		,     lowerCamelCase__     :   bool = False		,     **lowerCamelCase__     :   str		,     )      ->Union[str, Any]:



					'''simple docstring'''







					if self.speaker_embeddings is not None:
							os.makedirs(os.path.join(lowerCamelCase__		,     lowerCamelCase__		,     "v2"       )		,     exist_ok=lowerCamelCase__       )

							_UpperCAmelCase						:				Any	    =					{}

							_UpperCAmelCase						:				Tuple	    =					save_directory

							for prompt_key in self.speaker_embeddings:
									if prompt_key != "repo_or_path":
											_UpperCAmelCase						:				Optional[int]	    =					self._load_voice_preset(lowerCamelCase__       )

											_UpperCAmelCase						:				Tuple	    =					{}
											for key in self.speaker_embeddings[prompt_key]:
													np.save(
													    os.path.join(
													        embeddings_dict["repo_or_path"]		,     lowerCamelCase__		,     F"""{prompt_key}_{key}"""       )		,     voice_preset[key]		,     allow_pickle=lowerCamelCase__		,     )
													_UpperCAmelCase						:				int	    =					os.path.join(lowerCamelCase__		,     F"""{prompt_key}_{key}.npy"""       )

											_UpperCAmelCase						:				Optional[int]	    =					tmp_dict

							with open(os.path.join(lowerCamelCase__		,     lowerCamelCase__       )		,     "w"       ) as fp:
									json.dump(lowerCamelCase__		,     lowerCamelCase__       )

					super().save_pretrained(lowerCamelCase__		,     lowerCamelCase__		,     **lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   Dict		,     lowerCamelCase__     :   str = None		,     **lowerCamelCase__     :   Dict       )      ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					self.speaker_embeddings[voice_preset]

					_UpperCAmelCase						:				Union[str, Any]	    =					{}
					for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
							if key not in voice_preset_paths:
									raise ValueError(
									    F"""Voice preset unrecognized, missing {key} as a key in self.speaker_embeddings[{voice_preset}]."""       )

							_UpperCAmelCase						:				List[str]	    =					get_file_from_repo(
							    self.speaker_embeddings.get("repo_or_path"		,     "/"       )		,     voice_preset_paths[key]		,     subfolder=kwargs.pop("subfolder"		,     lowerCamelCase__       )		,     cache_dir=kwargs.pop("cache_dir"		,     lowerCamelCase__       )		,     force_download=kwargs.pop("force_download"		,     lowerCamelCase__       )		,     proxies=kwargs.pop("proxies"		,     lowerCamelCase__       )		,     resume_download=kwargs.pop("resume_download"		,     lowerCamelCase__       )		,     local_files_only=kwargs.pop("local_files_only"		,     lowerCamelCase__       )		,     use_auth_token=kwargs.pop("use_auth_token"		,     lowerCamelCase__       )		,     revision=kwargs.pop("revision"		,     lowerCamelCase__       )		,     )
							if path is None:
									raise ValueError(
									    F"""`{os.path.join(self.speaker_embeddings.get('repo_or_path'		,     '/'       )		,     voice_preset_paths[key]       )}` does not exists
                    , no preloaded voice preset will be used - Make sure to provide correct paths to the {voice_preset}
                    embeddings."""       )

							_UpperCAmelCase						:				Optional[Any]	    =					np.load(lowerCamelCase__       )

					return voice_preset_dict




			def 							lowerCAmelCase__			(						self     :   Tuple		,     lowerCamelCase__     :   Optional[dict] = None       )      ->Any:



					'''simple docstring'''







					for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
							if key not in voice_preset:
									raise ValueError(F"""Voice preset unrecognized, missing {key} as a key."""       )

							if not isinstance(voice_preset[key]		,     np.ndarray       ):
									raise ValueError(F"""{key} voice preset must be a {str(self.preset_shape[key]       )}D ndarray."""       )

							if len(voice_preset[key].shape       ) != self.preset_shape[key]:
									raise ValueError(F"""{key} voice preset must be a {str(self.preset_shape[key]       )}D ndarray."""       )






			def __call__(						self     :   int		,     lowerCamelCase__     :   Union[str, Any]=None		,     lowerCamelCase__     :   List[str]=None		,     lowerCamelCase__     :   Dict="pt"		,     lowerCamelCase__     :   List[Any]=2_56		,     lowerCamelCase__     :   Optional[Any]=False		,     lowerCamelCase__     :   Dict=True		,     lowerCamelCase__     :   List[Any]=False		,     **lowerCamelCase__     :   Tuple		,     )      ->Tuple:



					'''simple docstring'''







					if voice_preset is not None and not isinstance(lowerCamelCase__		,     lowerCamelCase__       ):
							if (
							    isinstance(lowerCamelCase__		,     lowerCamelCase__       )
							    and self.speaker_embeddings is not None
							    and voice_preset in self.speaker_embeddings
							):
									_UpperCAmelCase						:				Tuple	    =					self._load_voice_preset(lowerCamelCase__       )

							else:
									if isinstance(lowerCamelCase__		,     lowerCamelCase__       ) and not voice_preset.endswith(".npz"       ):
											_UpperCAmelCase						:				List[str]	    =					voice_preset + ".npz"

									_UpperCAmelCase						:				List[Any]	    =					np.load(lowerCamelCase__       )

					if voice_preset is not None:
							self._validate_voice_preset_dict(lowerCamelCase__		,     **lowerCamelCase__       )
							_UpperCAmelCase						:				Optional[Any]	    =					BatchFeature(data=lowerCamelCase__		,     tensor_type=lowerCamelCase__       )

					_UpperCAmelCase						:				int	    =					self.tokenizer(
					    lowerCamelCase__		,     return_tensors=lowerCamelCase__		,     padding="max_length"		,     max_length=lowerCamelCase__		,     return_attention_mask=lowerCamelCase__		,     return_token_type_ids=lowerCamelCase__		,     add_special_tokens=lowerCamelCase__		,     **lowerCamelCase__		,     )

					if voice_preset is not None:
							_UpperCAmelCase						:				int	    =					voice_preset

					return encoded_text







'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCamelCase__					=   logging.get_logger(__name__)

lowerCamelCase__					=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__					=   {
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase__					=   {
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

lowerCamelCase__					=   {
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Union[str, Any]				      =   VOCAB_FILES_NAMES
			lowerCAmelCase       :		Dict				      =   PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase       :		Tuple				      =   PRETRAINED_INIT_CONFIGURATION
			lowerCAmelCase       :		Optional[int]				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase       :		Any				      =   BertTokenizer




			def __init__(						self     :   int		,     lowerCamelCase__     :   Optional[Any]=None		,     lowerCamelCase__     :   Optional[int]=None		,     lowerCamelCase__     :   str=True		,     lowerCamelCase__     :   Tuple="[UNK]"		,     lowerCamelCase__     :   str="[SEP]"		,     lowerCamelCase__     :   Optional[Any]="[PAD]"		,     lowerCamelCase__     :   List[str]="[CLS]"		,     lowerCamelCase__     :   Union[str, Any]="[MASK]"		,     lowerCamelCase__     :   str=True		,     lowerCamelCase__     :   Dict=None		,     **lowerCamelCase__     :   Union[str, Any]		,     )      ->Tuple:



					'''simple docstring'''







					super().__init__(
					    lowerCamelCase__		,     tokenizer_file=lowerCamelCase__		,     do_lower_case=lowerCamelCase__		,     unk_token=lowerCamelCase__		,     sep_token=lowerCamelCase__		,     pad_token=lowerCamelCase__		,     cls_token=lowerCamelCase__		,     mask_token=lowerCamelCase__		,     tokenize_chinese_chars=lowerCamelCase__		,     strip_accents=lowerCamelCase__		,     **lowerCamelCase__		,     )

					_UpperCAmelCase						:				Optional[Any]	    =					json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
					if (
					    normalizer_state.get("lowercase"		,     lowerCamelCase__       ) != do_lower_case
					    or normalizer_state.get("strip_accents"		,     lowerCamelCase__       ) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,     lowerCamelCase__       ) != tokenize_chinese_chars
					):
							_UpperCAmelCase						:				str	    =					getattr(lowerCamelCase__		,     normalizer_state.pop("type"       )       )
							_UpperCAmelCase						:				Optional[Any]	    =					do_lower_case
							_UpperCAmelCase						:				Any	    =					strip_accents
							_UpperCAmelCase						:				List[Any]	    =					tokenize_chinese_chars
							_UpperCAmelCase						:				int	    =					normalizer_class(**lowerCamelCase__       )

					_UpperCAmelCase						:				Union[str, Any]	    =					do_lower_case




			def 							lowerCAmelCase__			(						self     :   List[str]		,     lowerCamelCase__     :   List[str]		,     lowerCamelCase__     :   str=None       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

					return output




			def 							lowerCAmelCase__			(						self     :   List[Any]		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Tuple	    =					[self.sep_token_id]
					_UpperCAmelCase						:				int	    =					[self.cls_token_id]
					if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






			def 							lowerCAmelCase__			(						self     :   List[Any]		,     lowerCamelCase__     :   str		,     lowerCamelCase__     :   Optional[str] = None       )      ->Tuple[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					self._tokenizer.model.save(lowerCamelCase__		,     name=lowerCamelCase__       )
					return tuple(lowerCamelCase__       )



