





"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


a_ =			logging.get_logger(__name__)

a_ =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a_ =			{
    """vocab_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/vocab.txt""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/vocab.txt""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"""
        ),
        """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt""",
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"""
        ),
        """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt""",
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"""
        ),
        """bert-base-multilingual-cased""": (
            """https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-cased""": (
            """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"""
        ),
    },
}

a_ =			{
    """bert-base-uncased""": 512,
    """bert-large-uncased""": 512,
    """bert-base-cased""": 512,
    """bert-large-cased""": 512,
    """bert-base-multilingual-uncased""": 512,
    """bert-base-multilingual-cased""": 512,
    """bert-base-chinese""": 512,
    """bert-base-german-cased""": 512,
    """bert-large-uncased-whole-word-masking""": 512,
    """bert-large-cased-whole-word-masking""": 512,
    """bert-large-uncased-whole-word-masking-finetuned-squad""": 512,
    """bert-large-cased-whole-word-masking-finetuned-squad""": 512,
    """bert-base-cased-finetuned-mrpc""": 512,
    """bert-base-german-dbmdz-cased""": 512,
    """bert-base-german-dbmdz-uncased""": 512,
    """TurkuNLP/bert-base-finnish-cased-v1""": 512,
    """TurkuNLP/bert-base-finnish-uncased-v1""": 512,
    """wietsedv/bert-base-dutch-cased""": 512,
}

a_ =			{
    """bert-base-uncased""": {"""do_lower_case""": True},
    """bert-large-uncased""": {"""do_lower_case""": True},
    """bert-base-cased""": {"""do_lower_case""": False},
    """bert-large-cased""": {"""do_lower_case""": False},
    """bert-base-multilingual-uncased""": {"""do_lower_case""": True},
    """bert-base-multilingual-cased""": {"""do_lower_case""": False},
    """bert-base-chinese""": {"""do_lower_case""": False},
    """bert-base-german-cased""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": False},
    """bert-base-cased-finetuned-mrpc""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-cased""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-uncased""": {"""do_lower_case""": True},
    """TurkuNLP/bert-base-finnish-cased-v1""": {"""do_lower_case""": False},
    """TurkuNLP/bert-base-finnish-uncased-v1""": {"""do_lower_case""": True},
    """wietsedv/bert-base-dutch-cased""": {"""do_lower_case""": False},
}






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 _lowerCamelCase        =     VOCAB_FILES_NAMES
 _lowerCamelCase        =     PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase        =     PRETRAINED_INIT_CONFIGURATION
 _lowerCamelCase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase        =     BertTokenizer

 def __init__(    self					,						__lowerCamelCase=None					,						__lowerCamelCase=None					,						__lowerCamelCase=True					,						__lowerCamelCase="[UNK]"					,						__lowerCamelCase="[SEP]"					,						__lowerCamelCase="[PAD]"					,						__lowerCamelCase="[CLS]"					,						__lowerCamelCase="[MASK]"					,						__lowerCamelCase=True					,						__lowerCamelCase=None					,						**__lowerCamelCase					,						):



     '''simple docstring'''




     super().__init__(
         __lowerCamelCase					,						tokenizer_file=__lowerCamelCase					,						do_lower_case=__lowerCamelCase					,						unk_token=__lowerCamelCase					,						sep_token=__lowerCamelCase					,						pad_token=__lowerCamelCase					,						cls_token=__lowerCamelCase					,						mask_token=__lowerCamelCase					,						tokenize_chinese_chars=__lowerCamelCase					,						strip_accents=__lowerCamelCase					,						**__lowerCamelCase					,						)

     __A    :						Optional[int]						 =     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
     if (
         normalizer_state.get('''lowercase'''					,						__lowerCamelCase  ) != do_lower_case
         or normalizer_state.get('''strip_accents'''					,						__lowerCamelCase  ) != strip_accents
         or normalizer_state.get('''handle_chinese_chars'''					,						__lowerCamelCase  ) != tokenize_chinese_chars
     ):
         __A    :						Union[str, Any]						 =     getattr(__lowerCamelCase					,						normalizer_state.pop('''type'''  )  )
         __A    :						int						 =     do_lower_case
         __A    :						Tuple						 =     strip_accents
         __A    :						int						 =     tokenize_chinese_chars
         __A    :						List[str]						 =     normalizer_class(**__lowerCamelCase  )

     __A    :						Dict						 =     do_lower_case

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase=None  ):



     '''simple docstring'''




     __A    :						List[Any]						 =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A    :						int						 =     [self.sep_token_id]
     __A    :						str						 =     [self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
     return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A    :						Dict						 =     self._tokenizer.model.save(__lowerCamelCase					,						name=__lowerCamelCase  )
     return tuple(__lowerCamelCase  )











"""simple docstring"""



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a_ =			logging.get_logger(__name__)

a_ =			{
    """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/config.json""",
    """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/config.json""",
    """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/config.json""",
    """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/config.json""",
    """bert-base-multilingual-uncased""": """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json""",
    """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json""",
    """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/config.json""",
    """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/config.json""",
    """bert-large-uncased-whole-word-masking""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-uncased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-base-cased-finetuned-mrpc""": """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json""",
    """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json""",
    """bert-base-german-dbmdz-uncased""": """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese""": """https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-cased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-uncased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json"""
    ),
    """wietsedv/bert-base-dutch-cased""": """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json""",
    # See all BERT models at https://huggingface.co/models?filter=bert
}






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 _lowerCamelCase        =     """bert"""

 def __init__(    self					,						__lowerCamelCase=3_0522					,						__lowerCamelCase=768					,						__lowerCamelCase=12					,						__lowerCamelCase=12					,						__lowerCamelCase=3072					,						__lowerCamelCase="gelu"					,						__lowerCamelCase=0.1					,						__lowerCamelCase=0.1					,						__lowerCamelCase=512					,						__lowerCamelCase=2					,						__lowerCamelCase=0.0_2					,						__lowerCamelCase=1e-1_2					,						__lowerCamelCase=0					,						__lowerCamelCase="absolute"					,						__lowerCamelCase=True					,						__lowerCamelCase=None					,						**__lowerCamelCase					,						):



     '''simple docstring'''




     super().__init__(pad_token_id=__lowerCamelCase					,						**__lowerCamelCase  )

     __A    :						Dict						 =     vocab_size
     __A    :						Any						 =     hidden_size
     __A    :						str						 =     num_hidden_layers
     __A    :						int						 =     num_attention_heads
     __A    :						Optional[int]						 =     hidden_act
     __A    :						List[Any]						 =     intermediate_size
     __A    :						Tuple						 =     hidden_dropout_prob
     __A    :						Optional[int]						 =     attention_probs_dropout_prob
     __A    :						Optional[int]						 =     max_position_embeddings
     __A    :						Optional[Any]						 =     type_vocab_size
     __A    :						Optional[Any]						 =     initializer_range
     __A    :						Dict						 =     layer_norm_eps
     __A    :						Any						 =     position_embedding_type
     __A    :						Optional[int]						 =     use_cache
     __A    :						str						 =     classifier_dropout






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 @property
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     if self.task == "multiple-choice":
         __A    :						str						 =     {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
     else:
         __A    :						Tuple						 =     {0: '''batch''', 1: '''sequence'''}
     return OrderedDict(
         [
             ('''input_ids''', dynamic_axis),
             ('''attention_mask''', dynamic_axis),
             ('''token_type_ids''', dynamic_axis),
         ]  )




