


'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


a__  :					List[str]        =       logging.get_logger(__name__)

a__  :					Optional[int]        =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

a__  :					Optional[int]        =       {
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

a__  :					str        =       {'mobilebert-uncased': 5_1_2}


a__  :					int        =       {}
class 					lowercase_     (    a__   ):
    __UpperCAmelCase   			=	VOCAB_FILES_NAMES
    __UpperCAmelCase   			=	PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase   			=	PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase   			=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase   			=	MobileBertTokenizer


    def __init__(     self  ,					a=None  ,					a=None  ,					a=True  ,					a="[UNK]"  ,					a="[SEP]"  ,					a="[PAD]"  ,					a="[CLS]"  ,					a="[MASK]"  ,					a=True  ,					a=None  ,					**a  ,					):
          super().__init__(
              a  ,					tokenizer_file=a  ,					do_lower_case=a  ,					unk_token=a  ,					sep_token=a  ,					pad_token=a  ,					cls_token=a  ,					mask_token=a  ,					tokenize_chinese_chars=a  ,					strip_accents=a  ,					**a  ,					)

          UpperCamelCase__										=				json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
          if (
              normalizer_state.get("lowercase"  ,					a    ) != do_lower_case
              or normalizer_state.get("strip_accents"  ,					a    ) != strip_accents
              or normalizer_state.get("handle_chinese_chars"  ,					a    ) != tokenize_chinese_chars
          ):
                UpperCamelCase__										=				getattr(a  ,					normalizer_state.pop("type"    )    )
                UpperCamelCase__										=				do_lower_case
                UpperCamelCase__										=				strip_accents
                UpperCamelCase__										=				tokenize_chinese_chars
                UpperCamelCase__										=				normalizer_class(**a    )

          UpperCamelCase__										=				do_lower_case


    def    __a							(     self  ,					a  ,					a=None    ):
          UpperCamelCase__										=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output


    def    __a							(     self  ,					a  ,					a = None    ):
          UpperCamelCase__										=				[self.sep_token_id]
          UpperCamelCase__										=				[self.cls_token_id]
          if token_ids_a is None:
                return len(cls + token_ids_a + sep    ) * [0]
          return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]


    def    __a							(     self  ,					a  ,					a = None    ):
          UpperCamelCase__										=				self._tokenizer.model.save(a  ,					name=a    )
          return tuple(a    )












'''simple docstring'''




lowercase__    :      Dict						=    {
    '''Pillow''': '''Pillow''',
    '''accelerate''': '''accelerate>=0.11.0''',
    '''compel''': '''compel==0.1.8''',
    '''black''': '''black~=23.1''',
    '''datasets''': '''datasets''',
    '''filelock''': '''filelock''',
    '''flax''': '''flax>=0.4.1''',
    '''hf-doc-builder''': '''hf-doc-builder>=0.3.0''',
    '''huggingface-hub''': '''huggingface-hub>=0.13.2''',
    '''requests-mock''': '''requests-mock==1.10.0''',
    '''importlib_metadata''': '''importlib_metadata''',
    '''invisible-watermark''': '''invisible-watermark''',
    '''isort''': '''isort>=5.5.4''',
    '''jax''': '''jax>=0.2.8,!=0.3.2''',
    '''jaxlib''': '''jaxlib>=0.1.65''',
    '''Jinja2''': '''Jinja2''',
    '''k-diffusion''': '''k-diffusion>=0.0.12''',
    '''torchsde''': '''torchsde''',
    '''note_seq''': '''note_seq''',
    '''librosa''': '''librosa''',
    '''numpy''': '''numpy''',
    '''omegaconf''': '''omegaconf''',
    '''parameterized''': '''parameterized''',
    '''protobuf''': '''protobuf>=3.20.3,<4''',
    '''pytest''': '''pytest''',
    '''pytest-timeout''': '''pytest-timeout''',
    '''pytest-xdist''': '''pytest-xdist''',
    '''ruff''': '''ruff>=0.0.241''',
    '''safetensors''': '''safetensors''',
    '''sentencepiece''': '''sentencepiece>=0.1.91,!=0.1.92''',
    '''scipy''': '''scipy''',
    '''onnx''': '''onnx''',
    '''regex''': '''regex!=2019.12.17''',
    '''requests''': '''requests''',
    '''tensorboard''': '''tensorboard''',
    '''torch''': '''torch>=1.4''',
    '''torchvision''': '''torchvision''',
    '''transformers''': '''transformers>=4.25.1''',
    '''urllib3''': '''urllib3<=2.0.0''',
}