




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


SCREAMING_SNAKE_CASE__       :      Tuple  =   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       :      Dict  =   {'vocab_file': 'vocab.txt'}

SCREAMING_SNAKE_CASE__       :      Tuple  =   {
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

SCREAMING_SNAKE_CASE__       :      Dict  =   {
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


SCREAMING_SNAKE_CASE__       :      int  =   {
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}




class 		UpperCamelCase__	(lowerCAmelCase__     ):

    '''simple docstring'''



    lowerCamelCase_      :	Optional[Any]	     =      VOCAB_FILES_NAMES
    lowerCamelCase_      :	List[str]	     =      PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase_      :	Optional[int]	     =      PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase_      :	Optional[int]	     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase_      :	Dict	     =      ConvBertTokenizer







    def __init__(    self     ,  UpperCamelCase__=None     ,  UpperCamelCase__=None     ,  UpperCamelCase__=True     ,  UpperCamelCase__="[UNK]"     ,  UpperCamelCase__="[SEP]"     ,  UpperCamelCase__="[PAD]"     ,  UpperCamelCase__="[CLS]"     ,  UpperCamelCase__="[MASK]"     ,  UpperCamelCase__=True     ,  UpperCamelCase__=None     ,  **UpperCamelCase__     ,  )     ->      List[str]:
           super().__init__(
               UpperCamelCase__     ,  tokenizer_file=UpperCamelCase__     ,  do_lower_case=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  sep_token=UpperCamelCase__     ,  pad_token=UpperCamelCase__     ,  cls_token=UpperCamelCase__     ,  mask_token=UpperCamelCase__     ,  tokenize_chinese_chars=UpperCamelCase__     ,  strip_accents=UpperCamelCase__     ,  **UpperCamelCase__     ,  )

           lowerCamelCase   :      int			 =					json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
           if (
               normalizer_state.get("lowercase"     ,  UpperCamelCase__     ) != do_lower_case
               or normalizer_state.get("strip_accents"     ,  UpperCamelCase__     ) != strip_accents
               or normalizer_state.get("handle_chinese_chars"     ,  UpperCamelCase__     ) != tokenize_chinese_chars
           ):
                  lowerCamelCase   :      Dict			 =					getattr(UpperCamelCase__     ,  normalizer_state.pop("type"     )     )
                  lowerCamelCase   :      Any			 =					do_lower_case
                  lowerCamelCase   :      Optional[Any]			 =					strip_accents
                  lowerCamelCase   :      Optional[int]			 =					tokenize_chinese_chars
                  lowerCamelCase   :      Optional[int]			 =					normalizer_class(**UpperCamelCase__     )

           lowerCamelCase   :      int			 =					do_lower_case







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__=None     )     ->      int:
           lowerCamelCase   :      Optional[int]			 =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      List[int]:
           lowerCamelCase   :      Any			 =					[self.sep_token_id]
           lowerCamelCase   :      Union[str, Any]			 =					[self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep     ) * [0]
           return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      Tuple[str]:
           lowerCamelCase   :      Optional[Any]			 =					self._tokenizer.model.save(UpperCamelCase__     ,  name=UpperCamelCase__     )
           return tuple(UpperCamelCase__     )








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A   =     {
    "configuration_distilbert": [
        "DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "DistilBertConfig",
        "DistilBertOnnxConfig",
    ],
    "tokenization_distilbert": ["DistilBertTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     ["DistilBertTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     [
			    "DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "DistilBertForMaskedLM",
			    "DistilBertForMultipleChoice",
			    "DistilBertForQuestionAnswering",
			    "DistilBertForSequenceClassification",
			    "DistilBertForTokenClassification",
			    "DistilBertModel",
			    "DistilBertPreTrainedModel",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     [
			    "TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFDistilBertForMaskedLM",
			    "TFDistilBertForMultipleChoice",
			    "TFDistilBertForQuestionAnswering",
			    "TFDistilBertForSequenceClassification",
			    "TFDistilBertForTokenClassification",
			    "TFDistilBertMainLayer",
			    "TFDistilBertModel",
			    "TFDistilBertPreTrainedModel",
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     [
			    "FlaxDistilBertForMaskedLM",
			    "FlaxDistilBertForMultipleChoice",
			    "FlaxDistilBertForQuestionAnswering",
			    "FlaxDistilBertForSequenceClassification",
			    "FlaxDistilBertForTokenClassification",
			    "FlaxDistilBertModel",
			    "FlaxDistilBertPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_distilbert import (
			    DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
			    DistilBertConfig,
			    DistilBertOnnxConfig,
			)
			from .tokenization_distilbert import DistilBertTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_distilbert_fast import DistilBertTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_distilbert import (
						    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    DistilBertForMaskedLM,
						    DistilBertForMultipleChoice,
						    DistilBertForQuestionAnswering,
						    DistilBertForSequenceClassification,
						    DistilBertForTokenClassification,
						    DistilBertModel,
						    DistilBertPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_distilbert import (
						    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFDistilBertForMaskedLM,
						    TFDistilBertForMultipleChoice,
						    TFDistilBertForQuestionAnswering,
						    TFDistilBertForSequenceClassification,
						    TFDistilBertForTokenClassification,
						    TFDistilBertMainLayer,
						    TFDistilBertModel,
						    TFDistilBertPreTrainedModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_distilbert import (
						    FlaxDistilBertForMaskedLM,
						    FlaxDistilBertForMultipleChoice,
						    FlaxDistilBertForQuestionAnswering,
						    FlaxDistilBertForSequenceClassification,
						    FlaxDistilBertForTokenClassification,
						    FlaxDistilBertModel,
						    FlaxDistilBertPreTrainedModel,
						)

else:
			import sys

			_A   =     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



