code: 
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_torch_available,
  )
  
  
  UpperCamelCase_	:    Optional[Any]  					=       {
      """auto_factory""": ["""get_values"""],
      """configuration_auto""": ["""ALL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CONFIG_MAPPING""", """MODEL_NAMES_MAPPING""", """AutoConfig"""],
      """feature_extraction_auto""": ["""FEATURE_EXTRACTOR_MAPPING""", """AutoFeatureExtractor"""],
      """image_processing_auto""": ["""IMAGE_PROCESSOR_MAPPING""", """AutoImageProcessor"""],
      """processing_auto""": ["""PROCESSOR_MAPPING""", """AutoProcessor"""],
      """tokenization_auto""": ["""TOKENIZER_MAPPING""", """AutoTokenizer"""],
  }
  
  try:
         if not is_torch_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         UpperCamelCase_	:    Optional[Any]  					=       [
             """MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_AUDIO_XVECTOR_MAPPING""",
             """MODEL_FOR_BACKBONE_MAPPING""",
             """MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING""",
             """MODEL_FOR_CAUSAL_LM_MAPPING""",
             """MODEL_FOR_CTC_MAPPING""",
             """MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING""",
             """MODEL_FOR_DEPTH_ESTIMATION_MAPPING""",
             """MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_IMAGE_SEGMENTATION_MAPPING""",
             """MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING""",
             """MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING""",
             """MODEL_FOR_MASKED_LM_MAPPING""",
             """MODEL_FOR_MASK_GENERATION_MAPPING""",
             """MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
             """MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
             """MODEL_FOR_OBJECT_DETECTION_MAPPING""",
             """MODEL_FOR_PRETRAINING_MAPPING""",
             """MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
             """MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING""",
             """MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
             """MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
             """MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING""",
             """MODEL_FOR_TEXT_ENCODING_MAPPING""",
             """MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING""",
             """MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_VISION_2_SEQ_MAPPING""",
             """MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING""",
             """MODEL_MAPPING""",
             """MODEL_WITH_LM_HEAD_MAPPING""",
             """MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING""",
             """MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING""",
             """AutoModel""",
             """AutoBackbone""",
             """AutoModelForAudioClassification""",
             """AutoModelForAudioFrameClassification""",
             """AutoModelForAudioXVector""",
             """AutoModelForCausalLM""",
             """AutoModelForCTC""",
             """AutoModelForDepthEstimation""",
             """AutoModelForImageClassification""",
             """AutoModelForImageSegmentation""",
             """AutoModelForInstanceSegmentation""",
             """AutoModelForMaskGeneration""",
             """AutoModelForTextEncoding""",
             """AutoModelForMaskedImageModeling""",
             """AutoModelForMaskedLM""",
             """AutoModelForMultipleChoice""",
             """AutoModelForNextSentencePrediction""",
             """AutoModelForObjectDetection""",
             """AutoModelForPreTraining""",
             """AutoModelForQuestionAnswering""",
             """AutoModelForSemanticSegmentation""",
             """AutoModelForSeq2SeqLM""",
             """AutoModelForSequenceClassification""",
             """AutoModelForSpeechSeq2Seq""",
             """AutoModelForTableQuestionAnswering""",
             """AutoModelForTokenClassification""",
             """AutoModelForUniversalSegmentation""",
             """AutoModelForVideoClassification""",
             """AutoModelForVision2Seq""",
             """AutoModelForVisualQuestionAnswering""",
             """AutoModelForDocumentQuestionAnswering""",
             """AutoModelWithLMHead""",
             """AutoModelForZeroShotImageClassification""",
             """AutoModelForZeroShotObjectDetection""",
         ]
  
  try:
         if not is_tf_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         UpperCamelCase_	:    Tuple  					=       [
             """TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
             """TF_MODEL_FOR_CAUSAL_LM_MAPPING""",
             """TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
             """TF_MODEL_FOR_MASK_GENERATION_MAPPING""",
             """TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING""",
             """TF_MODEL_FOR_MASKED_LM_MAPPING""",
             """TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
             """TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
             """TF_MODEL_FOR_PRETRAINING_MAPPING""",
             """TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
             """TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING""",
             """TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING""",
             """TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
             """TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
             """TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
             """TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING""",
             """TF_MODEL_FOR_TEXT_ENCODING_MAPPING""",
             """TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
             """TF_MODEL_FOR_VISION_2_SEQ_MAPPING""",
             """TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING""",
             """TF_MODEL_MAPPING""",
             """TF_MODEL_WITH_LM_HEAD_MAPPING""",
             """TFAutoModel""",
             """TFAutoModelForAudioClassification""",
             """TFAutoModelForCausalLM""",
             """TFAutoModelForImageClassification""",
             """TFAutoModelForMaskedImageModeling""",
             """TFAutoModelForMaskedLM""",
             """TFAutoModelForMaskGeneration""",
             """TFAutoModelForMultipleChoice""",
             """TFAutoModelForNextSentencePrediction""",
             """TFAutoModelForPreTraining""",
             """TFAutoModelForDocumentQuestionAnswering""",
             """TFAutoModelForQuestionAnswering""",
             """TFAutoModelForSemanticSegmentation""",
             """TFAutoModelForSeq2SeqLM""",
             """TFAutoModelForSequenceClassification""",
             """TFAutoModelForSpeechSeq2Seq""",
             """TFAutoModelForTableQuestionAnswering""",
             """TFAutoModelForTextEncoding""",
             """TFAutoModelForTokenClassification""",
             """TFAutoModelForVision2Seq""",
             """TFAutoModelForZeroShotImageClassification""",
             """TFAutoModelWithLMHead""",
         ]
  
  try:
         if not is_flax_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         UpperCamelCase_	:    Optional[int]  					=       [
             """FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
             """FLAX_MODEL_FOR_CAUSAL_LM_MAPPING""",
             """FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
             """FLAX_MODEL_FOR_MASKED_LM_MAPPING""",
             """FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
             """FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
             """FLAX_MODEL_FOR_PRETRAINING_MAPPING""",
             """FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
             """FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
             """FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
             """FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
             """FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
             """FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING""",
             """FLAX_MODEL_MAPPING""",
             """FlaxAutoModel""",
             """FlaxAutoModelForCausalLM""",
             """FlaxAutoModelForImageClassification""",
             """FlaxAutoModelForMaskedLM""",
             """FlaxAutoModelForMultipleChoice""",
             """FlaxAutoModelForNextSentencePrediction""",
             """FlaxAutoModelForPreTraining""",
             """FlaxAutoModelForQuestionAnswering""",
             """FlaxAutoModelForSeq2SeqLM""",
             """FlaxAutoModelForSequenceClassification""",
             """FlaxAutoModelForSpeechSeq2Seq""",
             """FlaxAutoModelForTokenClassification""",
             """FlaxAutoModelForVision2Seq""",
         ]
  
  
  if TYPE_CHECKING:
         from .auto_factory import get_values
         from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, MODEL_NAMES_MAPPING, AutoConfig
         from .feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING, AutoFeatureExtractor
         from .image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoImageProcessor
         from .processing_auto import PROCESSOR_MAPPING, AutoProcessor
         from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
  
         try:
                if not is_torch_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_auto import (
                    MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                    MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
                    MODEL_FOR_AUDIO_XVECTOR_MAPPING,
                    MODEL_FOR_BACKBONE_MAPPING,
                    MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
                    MODEL_FOR_CAUSAL_LM_MAPPING,
                    MODEL_FOR_CTC_MAPPING,
                    MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
                    MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
                    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
                    MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
                    MODEL_FOR_MASK_GENERATION_MAPPING,
                    MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
                    MODEL_FOR_MASKED_LM_MAPPING,
                    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                    MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                    MODEL_FOR_OBJECT_DETECTION_MAPPING,
                    MODEL_FOR_PRETRAINING_MAPPING,
                    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                    MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
                    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
                    MODEL_FOR_TEXT_ENCODING_MAPPING,
                    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                    MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
                    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
                    MODEL_FOR_VISION_2_SEQ_MAPPING,
                    MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
                    MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
                    MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
                    MODEL_MAPPING,
                    MODEL_WITH_LM_HEAD_MAPPING,
                    AutoBackbone,
                    AutoModel,
                    AutoModelForAudioClassification,
                    AutoModelForAudioFrameClassification,
                    AutoModelForAudioXVector,
                    AutoModelForCausalLM,
                    AutoModelForCTC,
                    AutoModelForDepthEstimation,
                    AutoModelForDocumentQuestionAnswering,
                    AutoModelForImageClassification,
                    AutoModelForImageSegmentation,
                    AutoModelForInstanceSegmentation,
                    AutoModelForMaskedImageModeling,
                    AutoModelForMaskedLM,
                    AutoModelForMaskGeneration,
                    AutoModelForMultipleChoice,
                    AutoModelForNextSentencePrediction,
                    AutoModelForObjectDetection,
                    AutoModelForPreTraining,
                    AutoModelForQuestionAnswering,
                    AutoModelForSemanticSegmentation,
                    AutoModelForSeqaSeqLM,
                    AutoModelForSequenceClassification,
                    AutoModelForSpeechSeqaSeq,
                    AutoModelForTableQuestionAnswering,
                    AutoModelForTextEncoding,
                    AutoModelForTokenClassification,
                    AutoModelForUniversalSegmentation,
                    AutoModelForVideoClassification,
                    AutoModelForVisionaSeq,
                    AutoModelForVisualQuestionAnswering,
                    AutoModelForZeroShotImageClassification,
                    AutoModelForZeroShotObjectDetection,
                    AutoModelWithLMHead,
                )
  
         try:
                if not is_tf_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_tf_auto import (
                    TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                    TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
                    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                    TF_MODEL_FOR_MASK_GENERATION_MAPPING,
                    TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
                    TF_MODEL_FOR_MASKED_LM_MAPPING,
                    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                    TF_MODEL_FOR_PRETRAINING_MAPPING,
                    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                    TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
                    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                    TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
                    TF_MODEL_FOR_TEXT_ENCODING_MAPPING,
                    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                    TF_MODEL_FOR_VISION_2_SEQ_MAPPING,
                    TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
                    TF_MODEL_MAPPING,
                    TF_MODEL_WITH_LM_HEAD_MAPPING,
                    TFAutoModel,
                    TFAutoModelForAudioClassification,
                    TFAutoModelForCausalLM,
                    TFAutoModelForDocumentQuestionAnswering,
                    TFAutoModelForImageClassification,
                    TFAutoModelForMaskedImageModeling,
                    TFAutoModelForMaskedLM,
                    TFAutoModelForMaskGeneration,
                    TFAutoModelForMultipleChoice,
                    TFAutoModelForNextSentencePrediction,
                    TFAutoModelForPreTraining,
                    TFAutoModelForQuestionAnswering,
                    TFAutoModelForSemanticSegmentation,
                    TFAutoModelForSeqaSeqLM,
                    TFAutoModelForSequenceClassification,
                    TFAutoModelForSpeechSeqaSeq,
                    TFAutoModelForTableQuestionAnswering,
                    TFAutoModelForTextEncoding,
                    TFAutoModelForTokenClassification,
                    TFAutoModelForVisionaSeq,
                    TFAutoModelForZeroShotImageClassification,
                    TFAutoModelWithLMHead,
                )
  
         try:
                if not is_flax_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_flax_auto import (
                    FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                    FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
                    FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                    FLAX_MODEL_FOR_MASKED_LM_MAPPING,
                    FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                    FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                    FLAX_MODEL_FOR_PRETRAINING_MAPPING,
                    FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                    FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                    FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                    FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                    FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                    FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING,
                    FLAX_MODEL_MAPPING,
                    FlaxAutoModel,
                    FlaxAutoModelForCausalLM,
                    FlaxAutoModelForImageClassification,
                    FlaxAutoModelForMaskedLM,
                    FlaxAutoModelForMultipleChoice,
                    FlaxAutoModelForNextSentencePrediction,
                    FlaxAutoModelForPreTraining,
                    FlaxAutoModelForQuestionAnswering,
                    FlaxAutoModelForSeqaSeqLM,
                    FlaxAutoModelForSequenceClassification,
                    FlaxAutoModelForSpeechSeqaSeq,
                    FlaxAutoModelForTokenClassification,
                    FlaxAutoModelForVisionaSeq,
                )
  
  else:
         import sys
  
         UpperCamelCase_	:    Optional[int]  					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
  
  
  
  
  
style_context: 
  import os
  import unittest
  
  from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  class      __lowercase	(	__snake_case      ,   unittest.TestCase					):
  	_A   			=      BertweetTokenizer
  	_A   			=      False
  
  
  
  
  
  
  	def 			_a(self							:   Tuple						)   -> Any:
  							super().setUp()
  
  							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  							_lowercase					:  Tuple    			=      ["I", "m", "V@@", "R@@", "r", "e@@"]
  							_lowercase					:  Tuple    			=      dict(zip(snake_case			,  range(len(snake_case						)						)						)						)
  							_lowercase					:  Tuple    			=      ["#version: 0.2", "a m</w>"]
  							_lowercase					:  Optional[Any]    			=      {"unk_token": "<unk>"}
  
  							_lowercase					:  int    			=      os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["vocab_file"]						)
  							_lowercase					:  Optional[int]    			=      os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["merges_file"]						)
  							with open(self.vocab_file			,  "w"			,  encoding="utf-8"						) as fp:
  													for token in vocab_tokens:
  																			fp.write(F"""{token} {vocab_tokens[token]}\n"""						)
  							with open(self.merges_file			,  "w"			,  encoding="utf-8"						) as fp:
  													fp.write("\n".join(snake_case						)						)
  
  
  
  
  
  
  	def 			_a(self							:   List[Any]			,  **snake_case							:   Tuple						)   -> Union[str, Any]:
  							kwargs.update(self.special_tokens_map						)
  							return BertweetTokenizer.from_pretrained(self.tmpdirname			,  **snake_case						)
  
  
  
  
  
  
  	def 			_a(self							:   Any			,  snake_case							:   Any						)   -> List[Any]:
  							_lowercase					:  Tuple    			=      "I am VinAI Research"
  							_lowercase					:  Optional[int]    			=      "I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
  							return input_text, output_text
  
  
  
  
  
  
  	def 			_a(self							:   str						)   -> Optional[int]:
  							_lowercase					:  str    			=      BertweetTokenizer(self.vocab_file			,  self.merges_file			,  **self.special_tokens_map						)
  							_lowercase					:  Optional[Any]    			=      "I am VinAI Research"
  							_lowercase					:  str    			=      "I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
  							_lowercase					:  Tuple    			=      tokenizer.tokenize(snake_case						)
  							self.assertListEqual(snake_case			,  snake_case						)
  
  							_lowercase					:  str    			=      tokens + [tokenizer.unk_token]
  
  							_lowercase					:  str    			=      [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
  							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case						)			,  snake_case						)
  
  
  
  
  