
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__SCREAMING_SNAKE_CASE		:  Union[str, Any]        =      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE		:  List[str]        =      {'vocab_file': 'vocab.txt'}

__SCREAMING_SNAKE_CASE		:  Tuple        =      {
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

__SCREAMING_SNAKE_CASE		:  Dict        =      {
    'tau/splinter-base': 512,
    'tau/splinter-base-qass': 512,
    'tau/splinter-large': 512,
    'tau/splinter-large-qass': 512,
}

__SCREAMING_SNAKE_CASE		:  str        =      {
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}
class  SCREAMING_SNAKE_CASE__       (  __UpperCamelCase   ):
	lowercase__	    =							VOCAB_FILES_NAMES
	lowercase__	    =							PRETRAINED_VOCAB_FILES_MAP
	lowercase__	    =							PRETRAINED_INIT_CONFIGURATION
	lowercase__	    =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__	    =							SplinterTokenizer
	def __init__(				self  ,    __UpperCamelCase=None  ,    __UpperCamelCase=None  ,    __UpperCamelCase=True  ,    __UpperCamelCase="[UNK]"  ,    __UpperCamelCase="[SEP]"  ,    __UpperCamelCase="[PAD]"  ,    __UpperCamelCase="[CLS]"  ,    __UpperCamelCase="[MASK]"  ,    __UpperCamelCase="[QUESTION]"  ,    __UpperCamelCase=True  ,    __UpperCamelCase=None  ,    **__UpperCamelCase  ,    ):



					'''simple docstring'''






					super().__init__(
					    __UpperCamelCase  ,    tokenizer_file=__UpperCamelCase  ,    do_lower_case=__UpperCamelCase  ,    unk_token=__UpperCamelCase  ,    sep_token=__UpperCamelCase  ,    pad_token=__UpperCamelCase  ,    cls_token=__UpperCamelCase  ,    mask_token=__UpperCamelCase  ,    tokenize_chinese_chars=__UpperCamelCase  ,    strip_accents=__UpperCamelCase  ,    additional_special_tokens=(question_token,)  ,    **__UpperCamelCase  ,    )

					__a       :     List[str]								=	json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
					if (
					    pre_tok_state.get("""lowercase"""  ,    __UpperCamelCase			) != do_lower_case
					    or pre_tok_state.get("""strip_accents"""  ,    __UpperCamelCase			) != strip_accents
					):
									__a       :     Dict								=	getattr(__UpperCamelCase  ,    pre_tok_state.pop("""type"""			)			)
									__a       :     int								=	do_lower_case
									__a       :     Dict								=	strip_accents
									__a       :     Union[str, Any]								=	pre_tok_class(**__UpperCamelCase			)

					__a       :     Optional[int]								=	do_lower_case
	@property
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					return self.convert_tokens_to_ids(self.question_token			)
	def __lowerCamelCase    (				self  ,    __UpperCamelCase  ,    __UpperCamelCase = None			):



					'''simple docstring'''






					if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					__a       :     Optional[Any]								=	[self.cls_token_id]
					__a       :     int								=	[self.sep_token_id]
					__a       :     Dict								=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""			)]
					if self.padding_side == "right":
									# Input is question-then-context
									return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
					else:
									# Input is context-then-question
									return cls + token_ids_a + sep + token_ids_a + question_suffix + sep
	def __lowerCamelCase    (				self  ,    __UpperCamelCase  ,    __UpperCamelCase = None			):



					'''simple docstring'''






					__a       :     List[str]								=	[self.sep_token_id]
					__a       :     List[Any]								=	[self.cls_token_id]
					__a       :     List[Any]								=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""			)]
					if token_ids_a is None:
									return len(cls + token_ids_a + sep			) * [0]

					if self.padding_side == "right":
									# Input is question-then-context
									return len(cls + token_ids_a + question_suffix + sep			) * [0] + len(token_ids_a + sep			) * [1]
					else:
									# Input is context-then-question
									return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + question_suffix + sep			) * [1]

	def __lowerCamelCase    (				self  ,    __UpperCamelCase  ,    __UpperCamelCase = None			):



					'''simple docstring'''






					__a       :     Optional[Any]								=	self._tokenizer.model.save(__UpperCamelCase  ,    name=__UpperCamelCase			)
					return tuple(__UpperCamelCase			)

'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


__SCREAMING_SNAKE_CASE		:  Tuple        =      {
    'configuration_segformer': ['SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'SegformerConfig', 'SegformerOnnxConfig']
}

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__SCREAMING_SNAKE_CASE		:  Dict        =      ['SegformerFeatureExtractor']
					__SCREAMING_SNAKE_CASE		:  str        =      ['SegformerImageProcessor']

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__SCREAMING_SNAKE_CASE		:  List[str]        =      [
					    'SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'SegformerDecodeHead',
					    'SegformerForImageClassification',
					    'SegformerForSemanticSegmentation',
					    'SegformerLayer',
					    'SegformerModel',
					    'SegformerPreTrainedModel',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__SCREAMING_SNAKE_CASE		:  List[str]        =      [
					    'TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'TFSegformerDecodeHead',
					    'TFSegformerForImageClassification',
					    'TFSegformerForSemanticSegmentation',
					    'TFSegformerModel',
					    'TFSegformerPreTrainedModel',
					]


if TYPE_CHECKING:
					from .configuration_segformer import SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, SegformerConfig, SegformerOnnxConfig

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_segformer import SegformerFeatureExtractor
										from .image_processing_segformer import SegformerImageProcessor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_segformer import (
										    SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    SegformerDecodeHead,
										    SegformerForImageClassification,
										    SegformerForSemanticSegmentation,
										    SegformerLayer,
										    SegformerModel,
										    SegformerPreTrainedModel,
										)
					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_segformer import (
										    TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFSegformerDecodeHead,
										    TFSegformerForImageClassification,
										    TFSegformerForSemanticSegmentation,
										    TFSegformerModel,
										    TFSegformerPreTrainedModel,
										)

else:
					import sys

					__SCREAMING_SNAKE_CASE		:  Optional[Any]        =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)