


"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__SCREAMING_SNAKE_CASE					=						logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE					=						{"""vocab_file""": """vocab.txt"""}

__SCREAMING_SNAKE_CASE					=						{
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

__SCREAMING_SNAKE_CASE					=						{
    """tau/splinter-base""": 5_12,
    """tau/splinter-base-qass""": 5_12,
    """tau/splinter-large""": 5_12,
    """tau/splinter-large-qass""": 5_12,
}

__SCREAMING_SNAKE_CASE					=						{
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}





class __snake_case		(					_SCREAMING_SNAKE_CASE      ):







			"""simple docstring"""




			lowerCAmelCase_				:   Tuple     	=			VOCAB_FILES_NAMES
			lowerCAmelCase_				:   List[str]     	=			PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_				:   Any     	=			PRETRAINED_INIT_CONFIGURATION
			lowerCAmelCase_				:   Optional[Any]     	=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_				:   Optional[int]     	=			SplinterTokenizer




			def __init__(		self							:Union[str, Any]				,  UpperCamelCase__							:Optional[int]=None				,  UpperCamelCase__							:Optional[Any]=None				,  UpperCamelCase__							:Union[str, Any]=True				,  UpperCamelCase__							:Dict="[UNK]"				,  UpperCamelCase__							:Union[str, Any]="[SEP]"				,  UpperCamelCase__							:List[Any]="[PAD]"				,  UpperCamelCase__							:int="[CLS]"				,  UpperCamelCase__							:str="[MASK]"				,  UpperCamelCase__							:Tuple="[QUESTION]"				,  UpperCamelCase__							:int=True				,  UpperCamelCase__							:int=None				,  **UpperCamelCase__							:Optional[int]				,  ):
										super().__init__(
										    UpperCamelCase__				,  tokenizer_file=UpperCamelCase__				,  do_lower_case=UpperCamelCase__				,  unk_token=UpperCamelCase__				,  sep_token=UpperCamelCase__				,  pad_token=UpperCamelCase__				,  cls_token=UpperCamelCase__				,  mask_token=UpperCamelCase__				,  tokenize_chinese_chars=UpperCamelCase__				,  strip_accents=UpperCamelCase__				,  additional_special_tokens=(question_token,)				,  **UpperCamelCase__				,  )

										_a          =				json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
										if (
										    pre_tok_state.get("lowercase"				,  UpperCamelCase__       ) != do_lower_case
										    or pre_tok_state.get("strip_accents"				,  UpperCamelCase__       ) != strip_accents
										):
																	_a          =				getattr(UpperCamelCase__				,  pre_tok_state.pop("type"       )       )
																	_a          =				do_lower_case
																	_a          =				strip_accents
																	_a          =				pre_tok_class(**UpperCamelCase__       )

										_a          =				do_lower_case




			@property
			def SCREAMING_SNAKE_CASE_      (		self							:str       ):
										return self.convert_tokens_to_ids(self.question_token       )




			def SCREAMING_SNAKE_CASE_      (		self							:Optional[Any]				,  UpperCamelCase__							:List[int]				,  UpperCamelCase__							:Optional[List[int]] = None       ):
										if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										_a          =				[self.cls_token_id]
										_a          =				[self.sep_token_id]
										_a          =				[self.question_token_id] + [self.convert_tokens_to_ids("."       )]
										if self.padding_side == "right":
																	# Input is question-then-context
																	return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
										else:
																	# Input is context-then-question
																	return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




			def SCREAMING_SNAKE_CASE_      (		self							:List[Any]				,  UpperCamelCase__							:List[int]				,  UpperCamelCase__							:Optional[List[int]] = None       ):
										_a          =				[self.sep_token_id]
										_a          =				[self.cls_token_id]
										_a          =				[self.question_token_id] + [self.convert_tokens_to_ids("."       )]
										if token_ids_a is None:
																	return len(cls + token_ids_a + sep       ) * [0]

										if self.padding_side == "right":
																	# Input is question-then-context
																	return len(cls + token_ids_a + question_suffix + sep       ) * [0] + len(token_ids_a + sep       ) * [1]
										else:
																	# Input is context-then-question
																	return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + question_suffix + sep       ) * [1]




			def SCREAMING_SNAKE_CASE_      (		self							:Optional[int]				,  UpperCamelCase__							:str				,  UpperCamelCase__							:Optional[str] = None       ):
										_a          =				self._tokenizer.model.save(UpperCamelCase__				,  name=UpperCamelCase__       )
										return tuple(UpperCamelCase__       )




"""simple docstring"""




import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin





class __snake_case		(					_SCREAMING_SNAKE_CASE      ):







   """simple docstring"""




   lowerCAmelCase_				:   str     	=			'AutoFeatureExtractor'
   lowerCAmelCase_				:   List[Any]     	=			'Speech2Text2Tokenizer'




   def __init__(		self							:int				,  UpperCamelCase__							:List[str]				,  UpperCamelCase__							:Union[str, Any]       ):
          super().__init__(UpperCamelCase__				,  UpperCamelCase__       )
          _a          =				self.feature_extractor
          _a          =				False




   def __call__(		self							:List[str]				,  *UpperCamelCase__							:List[str]				,  **UpperCamelCase__							:Dict       ):
          # For backward compatibility
          if self._in_target_context_manager:
                 return self.current_processor(*UpperCamelCase__				,  **UpperCamelCase__       )

          if "raw_speech" in kwargs:
                 warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."       )
                 _a          =				kwargs.pop("raw_speech"       )
          else:
                 _a          =				kwargs.pop("audio"				,  UpperCamelCase__       )
          _a          =				kwargs.pop("sampling_rate"				,  UpperCamelCase__       )
          _a          =				kwargs.pop("text"				,  UpperCamelCase__       )
          if len(UpperCamelCase__       ) > 0:
                 _a          =				args[0]
                 _a          =				args[1:]

          if audio is None and text is None:
                 raise ValueError("You need to specify either an `audio` or `text` input to process."       )

          if audio is not None:
                 _a          =				self.feature_extractor(UpperCamelCase__				,  *UpperCamelCase__				,  sampling_rate=UpperCamelCase__				,  **UpperCamelCase__       )
          if text is not None:
                 _a          =				self.tokenizer(UpperCamelCase__				,  **UpperCamelCase__       )

          if text is None:
                 return inputs
          elif audio is None:
                 return encodings
          else:
                 _a          =				encodings["input_ids"]
                 return inputs




   def SCREAMING_SNAKE_CASE_      (		self							:Union[str, Any]				,  *UpperCamelCase__							:int				,  **UpperCamelCase__							:Optional[Any]       ):
          return self.tokenizer.batch_decode(*UpperCamelCase__				,  **UpperCamelCase__       )




   def SCREAMING_SNAKE_CASE_      (		self							:Optional[Any]				,  *UpperCamelCase__							:List[Any]				,  **UpperCamelCase__							:List[Any]       ):
          return self.tokenizer.decode(*UpperCamelCase__				,  **UpperCamelCase__       )




   @contextmanager
   def SCREAMING_SNAKE_CASE_      (		self							:Dict       ):
          warnings.warn(
              "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
              "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
              "your audio inputs, or in a separate call."       )
          _a          =				True
          _a          =				self.tokenizer
          yield
          _a          =				self.feature_extractor
          _a          =				False
