code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_torch_available,
  )
  
  
  _lowerCamelCase							:		List[str]										=						{
      'auto_factory': ['get_values'],
      'configuration_auto': ['ALL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'CONFIG_MAPPING', 'MODEL_NAMES_MAPPING', 'AutoConfig'],
      'feature_extraction_auto': ['FEATURE_EXTRACTOR_MAPPING', 'AutoFeatureExtractor'],
      'image_processing_auto': ['IMAGE_PROCESSOR_MAPPING', 'AutoImageProcessor'],
      'processing_auto': ['PROCESSOR_MAPPING', 'AutoProcessor'],
      'tokenization_auto': ['TOKENIZER_MAPPING', 'AutoTokenizer'],
  }
  
  try:
  						if not is_torch_available():
  												raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  						pass
  else:
  						_lowerCamelCase							:		int										=						[
  						    'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_AUDIO_XVECTOR_MAPPING',
  						    'MODEL_FOR_BACKBONE_MAPPING',
  						    'MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING',
  						    'MODEL_FOR_CAUSAL_LM_MAPPING',
  						    'MODEL_FOR_CTC_MAPPING',
  						    'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING',
  						    'MODEL_FOR_DEPTH_ESTIMATION_MAPPING',
  						    'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING',
  						    'MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING',
  						    'MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING',
  						    'MODEL_FOR_MASKED_LM_MAPPING',
  						    'MODEL_FOR_MASK_GENERATION_MAPPING',
  						    'MODEL_FOR_MULTIPLE_CHOICE_MAPPING',
  						    'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING',
  						    'MODEL_FOR_OBJECT_DETECTION_MAPPING',
  						    'MODEL_FOR_PRETRAINING_MAPPING',
  						    'MODEL_FOR_QUESTION_ANSWERING_MAPPING',
  						    'MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING',
  						    'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING',
  						    'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING',
  						    'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING',
  						    'MODEL_FOR_TEXT_ENCODING_MAPPING',
  						    'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING',
  						    'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_VISION_2_SEQ_MAPPING',
  						    'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING',
  						    'MODEL_MAPPING',
  						    'MODEL_WITH_LM_HEAD_MAPPING',
  						    'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING',
  						    'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING',
  						    'AutoModel',
  						    'AutoBackbone',
  						    'AutoModelForAudioClassification',
  						    'AutoModelForAudioFrameClassification',
  						    'AutoModelForAudioXVector',
  						    'AutoModelForCausalLM',
  						    'AutoModelForCTC',
  						    'AutoModelForDepthEstimation',
  						    'AutoModelForImageClassification',
  						    'AutoModelForImageSegmentation',
  						    'AutoModelForInstanceSegmentation',
  						    'AutoModelForMaskGeneration',
  						    'AutoModelForTextEncoding',
  						    'AutoModelForMaskedImageModeling',
  						    'AutoModelForMaskedLM',
  						    'AutoModelForMultipleChoice',
  						    'AutoModelForNextSentencePrediction',
  						    'AutoModelForObjectDetection',
  						    'AutoModelForPreTraining',
  						    'AutoModelForQuestionAnswering',
  						    'AutoModelForSemanticSegmentation',
  						    'AutoModelForSeq2SeqLM',
  						    'AutoModelForSequenceClassification',
  						    'AutoModelForSpeechSeq2Seq',
  						    'AutoModelForTableQuestionAnswering',
  						    'AutoModelForTokenClassification',
  						    'AutoModelForUniversalSegmentation',
  						    'AutoModelForVideoClassification',
  						    'AutoModelForVision2Seq',
  						    'AutoModelForVisualQuestionAnswering',
  						    'AutoModelForDocumentQuestionAnswering',
  						    'AutoModelWithLMHead',
  						    'AutoModelForZeroShotImageClassification',
  						    'AutoModelForZeroShotObjectDetection',
  						]
  
  try:
  						if not is_tf_available():
  												raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  						pass
  else:
  						_lowerCamelCase							:		Optional[int]										=						[
  						    'TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING',
  						    'TF_MODEL_FOR_CAUSAL_LM_MAPPING',
  						    'TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
  						    'TF_MODEL_FOR_MASK_GENERATION_MAPPING',
  						    'TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING',
  						    'TF_MODEL_FOR_MASKED_LM_MAPPING',
  						    'TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING',
  						    'TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING',
  						    'TF_MODEL_FOR_PRETRAINING_MAPPING',
  						    'TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING',
  						    'TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING',
  						    'TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING',
  						    'TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING',
  						    'TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING',
  						    'TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING',
  						    'TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING',
  						    'TF_MODEL_FOR_TEXT_ENCODING_MAPPING',
  						    'TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING',
  						    'TF_MODEL_FOR_VISION_2_SEQ_MAPPING',
  						    'TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING',
  						    'TF_MODEL_MAPPING',
  						    'TF_MODEL_WITH_LM_HEAD_MAPPING',
  						    'TFAutoModel',
  						    'TFAutoModelForAudioClassification',
  						    'TFAutoModelForCausalLM',
  						    'TFAutoModelForImageClassification',
  						    'TFAutoModelForMaskedImageModeling',
  						    'TFAutoModelForMaskedLM',
  						    'TFAutoModelForMaskGeneration',
  						    'TFAutoModelForMultipleChoice',
  						    'TFAutoModelForNextSentencePrediction',
  						    'TFAutoModelForPreTraining',
  						    'TFAutoModelForDocumentQuestionAnswering',
  						    'TFAutoModelForQuestionAnswering',
  						    'TFAutoModelForSemanticSegmentation',
  						    'TFAutoModelForSeq2SeqLM',
  						    'TFAutoModelForSequenceClassification',
  						    'TFAutoModelForSpeechSeq2Seq',
  						    'TFAutoModelForTableQuestionAnswering',
  						    'TFAutoModelForTextEncoding',
  						    'TFAutoModelForTokenClassification',
  						    'TFAutoModelForVision2Seq',
  						    'TFAutoModelForZeroShotImageClassification',
  						    'TFAutoModelWithLMHead',
  						]
  
  try:
  						if not is_flax_available():
  												raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  						pass
  else:
  						_lowerCamelCase							:		Optional[int]										=						[
  						    'FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING',
  						    'FLAX_MODEL_FOR_CAUSAL_LM_MAPPING',
  						    'FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
  						    'FLAX_MODEL_FOR_MASKED_LM_MAPPING',
  						    'FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING',
  						    'FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING',
  						    'FLAX_MODEL_FOR_PRETRAINING_MAPPING',
  						    'FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING',
  						    'FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING',
  						    'FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING',
  						    'FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING',
  						    'FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING',
  						    'FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING',
  						    'FLAX_MODEL_MAPPING',
  						    'FlaxAutoModel',
  						    'FlaxAutoModelForCausalLM',
  						    'FlaxAutoModelForImageClassification',
  						    'FlaxAutoModelForMaskedLM',
  						    'FlaxAutoModelForMultipleChoice',
  						    'FlaxAutoModelForNextSentencePrediction',
  						    'FlaxAutoModelForPreTraining',
  						    'FlaxAutoModelForQuestionAnswering',
  						    'FlaxAutoModelForSeq2SeqLM',
  						    'FlaxAutoModelForSequenceClassification',
  						    'FlaxAutoModelForSpeechSeq2Seq',
  						    'FlaxAutoModelForTokenClassification',
  						    'FlaxAutoModelForVision2Seq',
  						]
  
  
  if TYPE_CHECKING:
  						from .auto_factory import get_values
  						from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, MODEL_NAMES_MAPPING, AutoConfig
  						from .feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING, AutoFeatureExtractor
  						from .image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoImageProcessor
  						from .processing_auto import PROCESSOR_MAPPING, AutoProcessor
  						from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
  
  						try:
  												if not is_torch_available():
  																		raise OptionalDependencyNotAvailable()
  						except OptionalDependencyNotAvailable:
  												pass
  						else:
  												from .modeling_auto import (
  												    MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_AUDIO_XVECTOR_MAPPING,
  												    MODEL_FOR_BACKBONE_MAPPING,
  												    MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
  												    MODEL_FOR_CAUSAL_LM_MAPPING,
  												    MODEL_FOR_CTC_MAPPING,
  												    MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
  												    MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
  												    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
  												    MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
  												    MODEL_FOR_MASK_GENERATION_MAPPING,
  												    MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
  												    MODEL_FOR_MASKED_LM_MAPPING,
  												    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
  												    MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
  												    MODEL_FOR_OBJECT_DETECTION_MAPPING,
  												    MODEL_FOR_PRETRAINING_MAPPING,
  												    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
  												    MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
  												    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
  												    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
  												    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
  												    MODEL_FOR_TEXT_ENCODING_MAPPING,
  												    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
  												    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_VISION_2_SEQ_MAPPING,
  												    MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
  												    MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
  												    MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
  												    MODEL_MAPPING,
  												    MODEL_WITH_LM_HEAD_MAPPING,
  												    AutoBackbone,
  												    AutoModel,
  												    AutoModelForAudioClassification,
  												    AutoModelForAudioFrameClassification,
  												    AutoModelForAudioXVector,
  												    AutoModelForCausalLM,
  												    AutoModelForCTC,
  												    AutoModelForDepthEstimation,
  												    AutoModelForDocumentQuestionAnswering,
  												    AutoModelForImageClassification,
  												    AutoModelForImageSegmentation,
  												    AutoModelForInstanceSegmentation,
  												    AutoModelForMaskedImageModeling,
  												    AutoModelForMaskedLM,
  												    AutoModelForMaskGeneration,
  												    AutoModelForMultipleChoice,
  												    AutoModelForNextSentencePrediction,
  												    AutoModelForObjectDetection,
  												    AutoModelForPreTraining,
  												    AutoModelForQuestionAnswering,
  												    AutoModelForSemanticSegmentation,
  												    AutoModelForSeqaSeqLM,
  												    AutoModelForSequenceClassification,
  												    AutoModelForSpeechSeqaSeq,
  												    AutoModelForTableQuestionAnswering,
  												    AutoModelForTextEncoding,
  												    AutoModelForTokenClassification,
  												    AutoModelForUniversalSegmentation,
  												    AutoModelForVideoClassification,
  												    AutoModelForVisionaSeq,
  												    AutoModelForVisualQuestionAnswering,
  												    AutoModelForZeroShotImageClassification,
  												    AutoModelForZeroShotObjectDetection,
  												    AutoModelWithLMHead,
  												)
  
  						try:
  												if not is_tf_available():
  																		raise OptionalDependencyNotAvailable()
  						except OptionalDependencyNotAvailable:
  												pass
  						else:
  												from .modeling_tf_auto import (
  												    TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
  												    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
  												    TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
  												    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
  												    TF_MODEL_FOR_MASK_GENERATION_MAPPING,
  												    TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
  												    TF_MODEL_FOR_MASKED_LM_MAPPING,
  												    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
  												    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
  												    TF_MODEL_FOR_PRETRAINING_MAPPING,
  												    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
  												    TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
  												    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
  												    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
  												    TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
  												    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
  												    TF_MODEL_FOR_TEXT_ENCODING_MAPPING,
  												    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
  												    TF_MODEL_FOR_VISION_2_SEQ_MAPPING,
  												    TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
  												    TF_MODEL_MAPPING,
  												    TF_MODEL_WITH_LM_HEAD_MAPPING,
  												    TFAutoModel,
  												    TFAutoModelForAudioClassification,
  												    TFAutoModelForCausalLM,
  												    TFAutoModelForDocumentQuestionAnswering,
  												    TFAutoModelForImageClassification,
  												    TFAutoModelForMaskedImageModeling,
  												    TFAutoModelForMaskedLM,
  												    TFAutoModelForMaskGeneration,
  												    TFAutoModelForMultipleChoice,
  												    TFAutoModelForNextSentencePrediction,
  												    TFAutoModelForPreTraining,
  												    TFAutoModelForQuestionAnswering,
  												    TFAutoModelForSemanticSegmentation,
  												    TFAutoModelForSeqaSeqLM,
  												    TFAutoModelForSequenceClassification,
  												    TFAutoModelForSpeechSeqaSeq,
  												    TFAutoModelForTableQuestionAnswering,
  												    TFAutoModelForTextEncoding,
  												    TFAutoModelForTokenClassification,
  												    TFAutoModelForVisionaSeq,
  												    TFAutoModelForZeroShotImageClassification,
  												    TFAutoModelWithLMHead,
  												)
  
  						try:
  												if not is_flax_available():
  																		raise OptionalDependencyNotAvailable()
  						except OptionalDependencyNotAvailable:
  												pass
  						else:
  												from .modeling_flax_auto import (
  												    FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
  												    FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
  												    FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
  												    FLAX_MODEL_FOR_MASKED_LM_MAPPING,
  												    FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
  												    FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
  												    FLAX_MODEL_FOR_PRETRAINING_MAPPING,
  												    FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
  												    FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
  												    FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
  												    FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
  												    FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
  												    FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING,
  												    FLAX_MODEL_MAPPING,
  												    FlaxAutoModel,
  												    FlaxAutoModelForCausalLM,
  												    FlaxAutoModelForImageClassification,
  												    FlaxAutoModelForMaskedLM,
  												    FlaxAutoModelForMultipleChoice,
  												    FlaxAutoModelForNextSentencePrediction,
  												    FlaxAutoModelForPreTraining,
  												    FlaxAutoModelForQuestionAnswering,
  												    FlaxAutoModelForSeqaSeqLM,
  												    FlaxAutoModelForSequenceClassification,
  												    FlaxAutoModelForSpeechSeqaSeq,
  												    FlaxAutoModelForTokenClassification,
  												    FlaxAutoModelForVisionaSeq,
  												)
  
  else:
  						import sys
  
  						_lowerCamelCase							:		Optional[int]										=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
  
  
  
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import normalizers
  
  from ...tokenization_utils import AddedToken
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_mpnet import MPNetTokenizer
  
  
  _lowerCamelCase							:		int										=						logging.get_logger(__name__)
  
  _lowerCamelCase							:		Any										=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}
  
  _lowerCamelCase							:		Dict										=						{
      'vocab_file': {
          'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt',
      },
      'tokenizer_file': {
          'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json',
      },
  }
  
  _lowerCamelCase							:		List[str]										=						{
      'microsoft/mpnet-base': 512,
  }
  
  _lowerCamelCase							:		Tuple										=						{
      'microsoft/mpnet-base': {'do_lower_case': True},
  }
  
  
  
  
  
  class   lowerCamelCase_							( UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				VOCAB_FILES_NAMES
  					lowercase__					=				PRETRAINED_VOCAB_FILES_MAP
  					lowercase__					=				PRETRAINED_INIT_CONFIGURATION
  					lowercase__					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  					lowercase__					=				MPNetTokenizer
  					lowercase__					=				["""input_ids""", """attention_mask"""]
  
  
  
  
  
  
  					def __init__(					self,						__UpperCamelCase=None,						__UpperCamelCase=None,						__UpperCamelCase=True,						__UpperCamelCase="<s>",						__UpperCamelCase="</s>",						__UpperCamelCase="</s>",						__UpperCamelCase="<s>",						__UpperCamelCase="[UNK]",						__UpperCamelCase="<pad>",						__UpperCamelCase="<mask>",						__UpperCamelCase=True,						__UpperCamelCase=None,						**__UpperCamelCase,						):
  							super().__init__(
  							    __UpperCamelCase,						tokenizer_file=__UpperCamelCase,						do_lower_case=__UpperCamelCase,						bos_token=__UpperCamelCase,						eos_token=__UpperCamelCase,						sep_token=__UpperCamelCase,						cls_token=__UpperCamelCase,						unk_token=__UpperCamelCase,						pad_token=__UpperCamelCase,						mask_token=__UpperCamelCase,						tokenize_chinese_chars=__UpperCamelCase,						strip_accents=__UpperCamelCase,						**__UpperCamelCase,						)
  
  							_snake_case			 = json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
  							if (
  							    pre_tok_state.get("lowercase",						__UpperCamelCase				) != do_lower_case
  							    or pre_tok_state.get("strip_accents",						__UpperCamelCase				) != strip_accents
  							):
  									_snake_case			 = getattr(__UpperCamelCase,						pre_tok_state.pop("type"				)				)
  									_snake_case			 = do_lower_case
  									_snake_case			 = strip_accents
  									_snake_case			 = pre_tok_class(**__UpperCamelCase				)
  
  							_snake_case			 = do_lower_case
  
  
  
  
  
  
  					@property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							if self._mask_token is None:
  									if self.verbose:
  											logger.error("Using mask_token, but it is not set yet."				)
  									return None
  							return str(self._mask_token				)
  
  
  
  
  
  
  					@mask_token.setter
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case			 = AddedToken(__UpperCamelCase,						lstrip=__UpperCamelCase,						rstrip=__UpperCamelCase				) if isinstance(__UpperCamelCase,						__UpperCamelCase				) else value
  							_snake_case			 = value
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase=None				):
  							_snake_case			 = [self.bos_token_id] + token_ids_a + [self.eos_token_id]
  							if token_ids_a is None:
  									return output
  
  							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = None				):
  							_snake_case			 = [self.sep_token_id]
  							_snake_case			 = [self.cls_token_id]
  
  							if token_ids_a is None:
  									return len(cls + token_ids_a + sep				) * [0]
  							return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = None				):
  							_snake_case			 = self._tokenizer.model.save(__UpperCamelCase,						name=__UpperCamelCase				)
  							return tuple(__UpperCamelCase				)
  
  
  
  
  
  