






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       """Wav2Vec2FeatureExtractor"""
				lowercase__											=       """AutoTokenizer"""




				def __init__(     self						:    List[Any],  lowerCamelCase						:    Optional[Any],  lowerCamelCase						:    Optional[Any]  ):

								'''simple docstring'''
								super().__init__(lowerCamelCase,  lowerCamelCase  )
								lowercase__    =			self.feature_extractor
								lowercase__    =			False




				@classmethod
				def 				lowercase__   (     cls						:    List[Any],  lowerCamelCase						:    Optional[Any],  **lowerCamelCase						:    Union[str, Any]  ):

								'''simple docstring'''
								try:
												return super().from_pretrained(lowerCamelCase,  **lowerCamelCase  )
								except OSError:
												warnings.warn(
												    F"""Loading a tokenizer inside {cls.__name__} from a config that does not"""
												    ''' include a `tokenizer_class` attribute is deprecated and will be '''
												    '''removed in v5. Please add `\'tokenizer_class\': \'Wav2Vec2CTCTokenizer\'`'''
												    ''' attribute to either your `config.json` or `tokenizer_config.json` '''
												    '''file to suppress this warning: ''',  lowerCamelCase,  )

												lowercase__    =			WavaVecaFeatureExtractor.from_pretrained(lowerCamelCase,  **lowerCamelCase  )
												lowercase__    =			WavaVecaCTCTokenizer.from_pretrained(lowerCamelCase,  **lowerCamelCase  )

												return cls(feature_extractor=lowerCamelCase,  tokenizer=lowerCamelCase  )




				def __call__(     self						:    str,  *lowerCamelCase						:    Dict,  **lowerCamelCase						:    Optional[int]  ):

								'''simple docstring'''
								# For backward compatibility
								if self._in_target_context_manager:
												return self.current_processor(*lowerCamelCase,  **lowerCamelCase  )

								if "raw_speech" in kwargs:
												warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''  )
												lowercase__    =			kwargs.pop('''raw_speech'''  )
								else:
												lowercase__    =			kwargs.pop('''audio''',  lowerCamelCase  )
								lowercase__    =			kwargs.pop('''sampling_rate''',  lowerCamelCase  )
								lowercase__    =			kwargs.pop('''text''',  lowerCamelCase  )
								if len(lowerCamelCase  ) > 0:
												lowercase__    =			args[0]
												lowercase__    =			args[1:]

								if audio is None and text is None:
												raise ValueError('''You need to specify either an `audio` or `text` input to process.'''  )

								if audio is not None:
												lowercase__    =			self.feature_extractor(lowerCamelCase,  *lowerCamelCase,  sampling_rate=lowerCamelCase,  **lowerCamelCase  )
								if text is not None:
												lowercase__    =			self.tokenizer(lowerCamelCase,  **lowerCamelCase  )

								if text is None:
												return inputs
								elif audio is None:
												return encodings
								else:
												lowercase__    =			encodings['''input_ids''']
												return inputs




				def 				lowercase__   (     self						:    List[Any],  *lowerCamelCase						:    Tuple,  **lowerCamelCase						:    List[str]  ):

								'''simple docstring'''
								# For backward compatibility
								if self._in_target_context_manager:
												return self.current_processor.pad(*lowerCamelCase,  **lowerCamelCase  )

								lowercase__    =			kwargs.pop('''input_features''',  lowerCamelCase  )
								lowercase__    =			kwargs.pop('''labels''',  lowerCamelCase  )
								if len(lowerCamelCase  ) > 0:
												lowercase__    =			args[0]
												lowercase__    =			args[1:]

								if input_features is not None:
												lowercase__    =			self.feature_extractor.pad(lowerCamelCase,  *lowerCamelCase,  **lowerCamelCase  )
								if labels is not None:
												lowercase__    =			self.tokenizer.pad(lowerCamelCase,  **lowerCamelCase  )

								if labels is None:
												return input_features
								elif input_features is None:
												return labels
								else:
												lowercase__    =			labels['''input_ids''']
												return input_features




				def 				lowercase__   (     self						:    List[Any],  *lowerCamelCase						:    Dict,  **lowerCamelCase						:    Tuple  ):

								'''simple docstring'''
								return self.tokenizer.batch_decode(*lowerCamelCase,  **lowerCamelCase  )




				def 				lowercase__   (     self						:    List[str],  *lowerCamelCase						:    Tuple,  **lowerCamelCase						:    Union[str, Any]  ):

								'''simple docstring'''
								return self.tokenizer.decode(*lowerCamelCase,  **lowerCamelCase  )




				@contextmanager
				def 				lowercase__   (     self						:    Optional[Any]  ):

								'''simple docstring'''
								warnings.warn(
								    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
								    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
								    '''your audio inputs, or in a separate call.'''  )
								lowercase__    =			True
								lowercase__    =			self.tokenizer
								yield
								lowercase__    =			self.feature_extractor
								lowercase__    =			False












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A__	:			List[Any]			   = logging.get_logger(__name__)

A__	:			str			   = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__	:			int			   = {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A__	:			Optional[Any]			   = {
    'distilbert-base-uncased': 5_12,
    'distilbert-base-uncased-distilled-squad': 5_12,
    'distilbert-base-cased': 5_12,
    'distilbert-base-cased-distilled-squad': 5_12,
    'distilbert-base-german-cased': 5_12,
    'distilbert-base-multilingual-cased': 5_12,
}


A__	:			List[str]			   = {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       VOCAB_FILES_NAMES
				lowercase__											=       PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=       PRETRAINED_INIT_CONFIGURATION
				lowercase__											=       ["""input_ids""", """attention_mask"""]
				lowercase__											=       DistilBertTokenizer




				def __init__(     self						:    List[Any],  lowerCamelCase						:    List[Any]=None,  lowerCamelCase						:    Dict=None,  lowerCamelCase						:    str=True,  lowerCamelCase						:    Optional[int]="[UNK]",  lowerCamelCase						:    Optional[Any]="[SEP]",  lowerCamelCase						:    List[Any]="[PAD]",  lowerCamelCase						:    Any="[CLS]",  lowerCamelCase						:    Union[str, Any]="[MASK]",  lowerCamelCase						:    str=True,  lowerCamelCase						:    int=None,  **lowerCamelCase						:    Union[str, Any],  ):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,  tokenizer_file=lowerCamelCase,  do_lower_case=lowerCamelCase,  unk_token=lowerCamelCase,  sep_token=lowerCamelCase,  pad_token=lowerCamelCase,  cls_token=lowerCamelCase,  mask_token=lowerCamelCase,  tokenize_chinese_chars=lowerCamelCase,  strip_accents=lowerCamelCase,  **lowerCamelCase,  )

								lowercase__    =			json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
								if (
								    normalizer_state.get('''lowercase''',  lowerCamelCase  ) != do_lower_case
								    or normalizer_state.get('''strip_accents''',  lowerCamelCase  ) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars''',  lowerCamelCase  ) != tokenize_chinese_chars
								):
												lowercase__    =			getattr(lowerCamelCase,  normalizer_state.pop('''type'''  )  )
												lowercase__    =			do_lower_case
												lowercase__    =			strip_accents
												lowercase__    =			tokenize_chinese_chars
												lowercase__    =			normalizer_class(**lowerCamelCase  )

								lowercase__    =			do_lower_case




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    Optional[Any],  lowerCamelCase						:    List[Any]=None  ):

								'''simple docstring'''
								lowercase__    =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''
								lowercase__    =			[self.sep_token_id]
								lowercase__    =			[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    str,  lowerCamelCase						:    Optional[str] = None  ):

								'''simple docstring'''
								lowercase__    =			self._tokenizer.model.save(lowerCamelCase,  name=lowerCamelCase  )
								return tuple(lowerCamelCase  )




