import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


UpperCamelCase__											=					logging.get_logger(__name__)

UpperCamelCase__											=					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__											=					{
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

UpperCamelCase__											=					{'mobilebert-uncased': 5_1_2}


UpperCamelCase__											=					{}





class      A							(				UpperCAmelCase_     ):
   __UpperCAmelCase : str      =			VOCAB_FILES_NAMES
   __UpperCAmelCase : str      =			PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase : Any      =			PRETRAINED_INIT_CONFIGURATION
   __UpperCAmelCase : Union[str, Any]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase : Dict      =			MobileBertTokenizer


   def __init__(self     :						List[str]       ,  __UpperCAmelCase     :						Tuple=None       ,  __UpperCAmelCase     :						str=None       ,  __UpperCAmelCase     :						str=True       ,  __UpperCAmelCase     :						Optional[int]="[UNK]"       ,  __UpperCAmelCase     :						List[str]="[SEP]"       ,  __UpperCAmelCase     :						Tuple="[PAD]"       ,  __UpperCAmelCase     :						int="[CLS]"       ,  __UpperCAmelCase     :						Tuple="[MASK]"       ,  __UpperCAmelCase     :						List[Any]=True       ,  __UpperCAmelCase     :						str=None       ,  **__UpperCAmelCase     :						Tuple       ,  )						->				Tuple:
         """simple docstring"""



         super().__init__(
             __UpperCAmelCase       ,  tokenizer_file=__UpperCAmelCase       ,  do_lower_case=__UpperCAmelCase       ,  unk_token=__UpperCAmelCase       ,  sep_token=__UpperCAmelCase       ,  pad_token=__UpperCAmelCase       ,  cls_token=__UpperCAmelCase       ,  mask_token=__UpperCAmelCase       ,  tokenize_chinese_chars=__UpperCAmelCase       ,  strip_accents=__UpperCAmelCase       ,  **__UpperCAmelCase       ,  )

         UpperCAmelCase__								=  json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
         if (
             normalizer_state.get("lowercase"       ,  __UpperCAmelCase		) != do_lower_case
             or normalizer_state.get("strip_accents"       ,  __UpperCAmelCase		) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,  __UpperCAmelCase		) != tokenize_chinese_chars
         ):
               UpperCAmelCase__								=  getattr(__UpperCAmelCase       ,  normalizer_state.pop("type"		)		)
               UpperCAmelCase__								=  do_lower_case
               UpperCAmelCase__								=  strip_accents
               UpperCAmelCase__								=  tokenize_chinese_chars
               UpperCAmelCase__								=  normalizer_class(**__UpperCAmelCase		)

         UpperCAmelCase__								=  do_lower_case


   def 			lowercase_      (self     :						Dict       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						Optional[Any]=None		)						->				List[str]:
         """simple docstring"""



         UpperCAmelCase__								=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output


   def 			lowercase_      (self     :						List[str]       ,  __UpperCAmelCase     :						List[int]       ,  __UpperCAmelCase     :						Optional[List[int]] = None		)						->				List[int]:
         """simple docstring"""



         UpperCAmelCase__								=  [self.sep_token_id]
         UpperCAmelCase__								=  [self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





   def 			lowercase_      (self     :						str       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						Optional[str] = None		)						->				Tuple[str]:
         """simple docstring"""



         UpperCAmelCase__								=  self._tokenizer.model.save(__UpperCAmelCase       ,  name=__UpperCAmelCase		)
         return tuple(__UpperCAmelCase		)





"""simple docstring"""




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin

@require_jieba
class 		a__						(      a_,			unittest.TestCase						):
   __lowerCAmelCase            =				CpmAntTokenizer
   __lowerCAmelCase            =				False
   def 			__magic_name__			(			self				):
          super().setUp()

          lowercase    :				List[str]       		=							[
              "<d>",
              "</d>",
              "<s>",
              "</s>",
              "</_>",
              "<unk>",
              "<pad>",
              "</n>",
              "我",
              "是",
              "C",
              "P",
              "M",
              "A",
              "n",
              "t",
          ]
          lowercase    :				Dict       		=							os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["vocab_file"]				)
          with open(self.vocab_file	,    "w"	,    encoding="utf-8"				) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)

   @tooslow
   def 			__magic_name__			(			self				):
          lowercase    :				Tuple       		=							CpmAntTokenizer.from_pretrained("openbmb/cpm-ant-10b"				)
          lowercase    :				str       		=							"今天天气真好！"
          lowercase    :				Optional[int]       		=							["今天", "天气", "真", "好", "！"]
          lowercase    :				int       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Any       		=							"今天天气真好！"
          lowercase    :				int       		=							[tokenizer.bos_token] + tokens

          lowercase    :				List[str]       		=							[6, 9_802, 14_962, 2_082, 831, 244]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)	,    _a				)

          lowercase    :				str       		=							tokenizer.decode(_a				)
          self.assertEqual(_a	,    _a				)


