



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


UpperCAmelCase__  =			logging.get_logger(__name__)

UpperCAmelCase__  =			{"vocab_file": "vocab.txt"}

UpperCAmelCase__  =			{
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

UpperCAmelCase__  =			{
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

UpperCAmelCase__  =			{
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}







class      __lowerCAmelCase       (     A		):
			UpperCamelCase     =	VOCAB_FILES_NAMES
			UpperCamelCase     =	PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase     =	PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase     =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase     =	SplinterTokenizer



			def __init__(		self  :    int					,						A  :    str=None					,						A  :    Union[str, Any]=None					,						A  :    List[str]=True					,						A  :    List[Any]="[UNK]"					,						A  :    Optional[int]="[SEP]"					,						A  :    Optional[Any]="[PAD]"					,						A  :    Optional[Any]="[CLS]"					,						A  :    str="[MASK]"					,						A  :    Dict="[QUESTION]"					,						A  :    Tuple=True					,						A  :    Any=None					,						**A  :    Optional[int]					,						)  ->       Union[str, Any]:



						"""simple docstring"""






						super().__init__(
						    A					,						tokenizer_file=A					,						do_lower_case=A					,						unk_token=A					,						sep_token=A					,						pad_token=A					,						cls_token=A					,						mask_token=A					,						tokenize_chinese_chars=A					,						strip_accents=A					,						additional_special_tokens=(question_token,)					,						**A					,						)

						_UpperCAmelCase									=  json.loads(self.backend_tokenizer.normalizer.__getstate__())
						if (
						    pre_tok_state.get('lowercase'					,						A) != do_lower_case
						    or pre_tok_state.get('strip_accents'					,						A) != strip_accents
						):
									_UpperCAmelCase									=  getattr(A					,						pre_tok_state.pop('type'))
									_UpperCAmelCase									=  do_lower_case
									_UpperCAmelCase									=  strip_accents
									_UpperCAmelCase									=  pre_tok_class(**A)

						_UpperCAmelCase									=  do_lower_case



			@property
			def        _lowerCamelCase			(		self  :    int)  ->       Any:



						"""simple docstring"""






						return self.convert_tokens_to_ids(self.question_token)



			def        _lowerCamelCase			(		self  :    Dict					,						A  :    List[int]					,						A  :    Optional[List[int]] = None)  ->       List[int]:



						"""simple docstring"""






						if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						_UpperCAmelCase									=  [self.cls_token_id]
						_UpperCAmelCase									=  [self.sep_token_id]
						_UpperCAmelCase									=  [self.question_token_id] + [self.convert_tokens_to_ids('.')]
						if self.padding_side == "right":
									# Input is question-then-context
									return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
						else:
									# Input is context-then-question
									return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



			def        _lowerCamelCase			(		self  :    str					,						A  :    List[int]					,						A  :    Optional[List[int]] = None)  ->       List[int]:



						"""simple docstring"""






						_UpperCAmelCase									=  [self.sep_token_id]
						_UpperCAmelCase									=  [self.cls_token_id]
						_UpperCAmelCase									=  [self.question_token_id] + [self.convert_tokens_to_ids('.')]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]

						if self.padding_side == "right":
									# Input is question-then-context
									return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
						else:
									# Input is context-then-question
									return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]



			def        _lowerCamelCase			(		self  :    Union[str, Any]					,						A  :    str					,						A  :    Optional[str] = None)  ->       Tuple[str]:



						"""simple docstring"""






						_UpperCAmelCase									=  self._tokenizer.model.save(A					,						name=A)
						return tuple(A)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__  =			logging.get_logger(__name__)

UpperCAmelCase__  =			{
    "dandelin/vilt-b32-mlm": "https://huggingface.co/dandelin/vilt-b32-mlm/blob/main/config.json"
}







class      __lowerCAmelCase       (     A		):
			UpperCamelCase     =	'''vilt'''



			def __init__(		self  :    List[Any]					,						A  :    Dict=3_05_22					,						A  :    Union[str, Any]=2					,						A  :    Tuple=2					,						A  :    List[Any]=40					,						A  :    Optional[int]=7_68					,						A  :    Any=12					,						A  :    Optional[int]=12					,						A  :    Optional[Any]=30_72					,						A  :    Optional[Any]="gelu"					,						A  :    str=0.0					,						A  :    Any=0.0					,						A  :    Union[str, Any]=0.0_2					,						A  :    Union[str, Any]=1E-12					,						A  :    Optional[Any]=3_84					,						A  :    Dict=32					,						A  :    int=3					,						A  :    int=True					,						A  :    Union[str, Any]=-1					,						A  :    int=False					,						A  :    int=-1					,						**A  :    Tuple					,						)  ->       Dict:



						"""simple docstring"""






						super().__init__(tie_word_embeddings=A					,						**A)

						_UpperCAmelCase									=  vocab_size
						_UpperCAmelCase									=  type_vocab_size
						_UpperCAmelCase									=  modality_type_vocab_size
						_UpperCAmelCase									=  max_position_embeddings

						_UpperCAmelCase									=  hidden_size
						_UpperCAmelCase									=  num_hidden_layers
						_UpperCAmelCase									=  num_attention_heads
						_UpperCAmelCase									=  intermediate_size
						_UpperCAmelCase									=  hidden_act
						_UpperCAmelCase									=  hidden_dropout_prob
						_UpperCAmelCase									=  attention_probs_dropout_prob
						_UpperCAmelCase									=  initializer_range
						_UpperCAmelCase									=  layer_norm_eps

						_UpperCAmelCase									=  image_size
						_UpperCAmelCase									=  patch_size
						_UpperCAmelCase									=  num_channels
						_UpperCAmelCase									=  qkv_bias
						_UpperCAmelCase									=  max_image_length
						_UpperCAmelCase									=  num_images
