


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__lowerCAmelCase		       =							logging.get_logger(__name__)

__lowerCAmelCase		       =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase		       =							{
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__lowerCAmelCase		       =							{
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

__lowerCAmelCase		       =							{
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}





class   __a			(    __UpperCamelCase      ):
 __lowercase :							int        = VOCAB_FILES_NAMES
 __lowercase :							str        = PRETRAINED_VOCAB_FILES_MAP
 __lowercase :							Tuple        = PRETRAINED_INIT_CONFIGURATION
 __lowercase :							str        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase :							str        = BertTokenizer
 def __init__(   self				, lowerCAmelCase__=None				, lowerCAmelCase__=None				, lowerCAmelCase__=True				, lowerCAmelCase__="[UNK]"				, lowerCAmelCase__="[SEP]"				, lowerCAmelCase__="[PAD]"				, lowerCAmelCase__="[CLS]"				, lowerCAmelCase__="[MASK]"				, lowerCAmelCase__=True				, lowerCAmelCase__=None				, **lowerCAmelCase__				, )					->				Union[str, Any]:




        '''simple docstring'''
        super().__init__(
            lowerCAmelCase__				, tokenizer_file=lowerCAmelCase__				, do_lower_case=lowerCAmelCase__				, unk_token=lowerCAmelCase__				, sep_token=lowerCAmelCase__				, pad_token=lowerCAmelCase__				, cls_token=lowerCAmelCase__				, mask_token=lowerCAmelCase__				, tokenize_chinese_chars=lowerCAmelCase__				, strip_accents=lowerCAmelCase__				, **lowerCAmelCase__				, )

        lowercase__:   Union[str, Any]  			=   json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('lowercase'				, lowerCAmelCase__ ) != do_lower_case
            or normalizer_state.get('strip_accents'				, lowerCAmelCase__ ) != strip_accents
            or normalizer_state.get('handle_chinese_chars'				, lowerCAmelCase__ ) != tokenize_chinese_chars
        ):
               lowercase__:   List[Any]  			=   getattr(lowerCAmelCase__				, normalizer_state.pop('type' ) )
               lowercase__:   Optional[Any]  			=   do_lower_case
               lowercase__:   List[Any]  			=   strip_accents
               lowercase__:   Any  			=   tokenize_chinese_chars
               lowercase__:   List[str]  			=   normalizer_class(**lowerCAmelCase__ )

        lowercase__:   List[str]  			=   do_lower_case
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__=None )					->				Union[str, Any]:




        '''simple docstring'''
        lowercase__:   Tuple  			=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				List[int]:




        '''simple docstring'''
        lowercase__:   Union[str, Any]  			=   [self.sep_token_id]
        lowercase__:   int  			=   [self.cls_token_id]
        if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				Tuple[str]:




        '''simple docstring'''
        lowercase__:   Tuple  			=   self._tokenizer.model.save(lowerCAmelCase__				, name=lowerCAmelCase__ )
        return tuple(lowerCAmelCase__ )






def    snake_case_     (  snake_case			,							snake_case     )  ->				str:
       if number < 0 or shift_amount < 0:
              raise ValueError('both inputs must be positive integers'     )

       lowercase__:   str  			=   str(bin(snake_case     )     )
       binary_number += "0" * shift_amount
       return binary_number




def    snake_case_     (  snake_case			,							snake_case     )  ->				str:
       if number < 0 or shift_amount < 0:
              raise ValueError('both inputs must be positive integers'     )

       lowercase__:   Optional[Any]  			=   str(bin(snake_case     )     )[2:]
       if shift_amount >= len(snake_case     ):
              return "0b0"
       lowercase__:   Optional[int]  			=   binary_number[: len(snake_case     ) - shift_amount]
       return "0b" + shifted_binary_number




def    snake_case_     (  snake_case			,							snake_case     )  ->				str:
       if number >= 0:  # Get binary representation of positive number
              lowercase__:   Union[str, Any]  			=   '0' + str(bin(snake_case     )     ).strip('-'     )[2:]
       else:  # Get binary (2's complement) representation of negative number
              lowercase__:   Dict  			=   len(bin(snake_case     )[3:]     )  # Find 2's complement of number
              lowercase__:   int  			=   bin(abs(snake_case     ) - (1 << binary_number_length)     )[3:]
              lowercase__:   Any  			=   (
                  '1' + '0' * (binary_number_length - len(snake_case     )) + binary_number
              )

       if shift_amount >= len(snake_case     ):
              return "0b" + binary_number[0] * len(snake_case     )
       return (
           "0b"
           + binary_number[0] * shift_amount
           + binary_number[: len(snake_case     ) - shift_amount]
       )


if __name__ == "__main__":
 import doctest

 doctest.testmod()


