code: 
  
  
  
  
  
  
  """simple docstring"""
  
  from collections import OrderedDict
  from typing import Mapping
  
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig
  from ...utils import logging
  
  
  lowercase__      =				logging.get_logger(__name__)
  
  lowercase__      =				{
      'google/mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/config.json'
  }
  
  
  class 	__lowerCamelCase       (				A__    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		a_			:       Optional[Any]      =							MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
  		a_			:       int      =							"""mobilebert"""
  
  
  
  
  
  		def __init__(     self							:			Union[str, Any]		,    a_							:			Optional[int]=3_05_22		,    a_							:			int=5_12		,    a_							:			str=24		,    a_							:			Optional[Any]=4		,    a_							:			Dict=5_12		,    a_							:			Optional[Any]="relu"		,    a_							:			Union[str, Any]=0.0		,    a_							:			Any=0.1		,    a_							:			Optional[Any]=5_12		,    a_							:			Union[str, Any]=2		,    a_							:			Tuple=0.02		,    a_							:			Optional[Any]=1e-1_2		,    a_							:			str=0		,    a_							:			List[str]=1_28		,    a_							:			Optional[int]=True		,    a_							:			Tuple=True		,    a_							:			Union[str, Any]=1_28		,    a_							:			Union[str, Any]=False		,    a_							:			List[Any]=True		,    a_							:			Optional[Any]=4		,    a_							:			Union[str, Any]="no_norm"		,    a_							:			Dict=True		,    a_							:			Dict=None		,    **a_							:			List[Any]		,    ):
  				super().__init__(pad_token_id=_lowerCAmelCase		,    **_lowerCAmelCase			)
  
  				lowerCAmelCase_     :							str             =    vocab_size
  				lowerCAmelCase_     :							str             =    hidden_size
  				lowerCAmelCase_     :							str             =    num_hidden_layers
  				lowerCAmelCase_     :							List[str]             =    num_attention_heads
  				lowerCAmelCase_     :							Optional[int]             =    hidden_act
  				lowerCAmelCase_     :							str             =    intermediate_size
  				lowerCAmelCase_     :							Union[str, Any]             =    hidden_dropout_prob
  				lowerCAmelCase_     :							List[Any]             =    attention_probs_dropout_prob
  				lowerCAmelCase_     :							Union[str, Any]             =    max_position_embeddings
  				lowerCAmelCase_     :							Optional[Any]             =    type_vocab_size
  				lowerCAmelCase_     :							Tuple             =    initializer_range
  				lowerCAmelCase_     :							Dict             =    layer_norm_eps
  				lowerCAmelCase_     :							List[Any]             =    embedding_size
  				lowerCAmelCase_     :							List[Any]             =    trigram_input
  				lowerCAmelCase_     :							Union[str, Any]             =    use_bottleneck
  				lowerCAmelCase_     :							int             =    intra_bottleneck_size
  				lowerCAmelCase_     :							List[Any]             =    use_bottleneck_attention
  				lowerCAmelCase_     :							Optional[Any]             =    key_query_shared_bottleneck
  				lowerCAmelCase_     :							List[str]             =    num_feedforward_networks
  				lowerCAmelCase_     :							Union[str, Any]             =    normalization_type
  				lowerCAmelCase_     :							List[str]             =    classifier_activation
  
  				if self.use_bottleneck:
  						lowerCAmelCase_     :							List[str]             =    intra_bottleneck_size
  				else:
  						lowerCAmelCase_     :							Optional[Any]             =    hidden_size
  
  				lowerCAmelCase_     :							List[Any]             =    classifier_dropout
  
  
  
  
  
  
  
  class 	__lowerCamelCase       (				A__    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		@property
  		def      lowerCamelCase (     self							:			List[str]			):
  				if self.task == "multiple-choice":
  						lowerCAmelCase_     :							int             =    {0: "batch", 1: "choice", 2: "sequence"}
  				else:
  						lowerCAmelCase_     :							Dict             =    {0: "batch", 1: "sequence"}
  				return OrderedDict(
  				    [
  				        ("input_ids", dynamic_axis),
  				        ("attention_mask", dynamic_axis),
  				        ("token_type_ids", dynamic_axis),
  				    ]			)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  """simple docstring"""
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class 	__lowerCamelCase       (				A__				,							unittest.TestCase    ):
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  		a_			:       List[str]      =							RealmTokenizer
  		a_			:       int      =							RealmTokenizerFast
  		a_			:       str      =							True
  		a_			:       Tuple      =							True
  		a_			:       List[Any]      =							filter_non_english
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				super().setUp()
  
  				lowerCAmelCase_     :							int             =    [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				lowerCAmelCase_     :							List[Any]             =    os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["vocab_file"]			)
  				with open(self.vocab_file		,    "w"		,    encoding="utf-8"			) as vocab_writer:
  						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]		,    a_							:			Any			):
  				lowerCAmelCase_     :							int             =    "UNwant\u00E9d,running"
  				lowerCAmelCase_     :							Optional[Any]             =    "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							Tuple             =    self.tokenizer_class(self.vocab_file			)
  
  				lowerCAmelCase_     :							List[str]             =    tokenizer.tokenize("UNwant\u00E9d,running"			)
  				self.assertListEqual(a_		,    ["un", "##want", "##ed", ",", "runn", "##ing"]			)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)		,    [9, 6, 7, 12, 10, 11]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				if not self.test_rust_tokenizer:
  						return
  
  				lowerCAmelCase_     :							str             =    self.get_tokenizer()
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer()
  
  				lowerCAmelCase_     :							Optional[Any]             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							List[Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							int             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							Any             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				# With lower casing
  				lowerCAmelCase_     :							Dict             =    self.get_tokenizer(do_lower_case=a_			)
  				lowerCAmelCase_     :							List[str]             =    self.get_rust_tokenizer(do_lower_case=a_			)
  
  				lowerCAmelCase_     :							Dict             =    "UNwant\u00E9d,running"
  
  				lowerCAmelCase_     :							Union[str, Any]             =    tokenizer.tokenize(a_			)
  				lowerCAmelCase_     :							Union[str, Any]             =    rust_tokenizer.tokenize(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							Optional[Any]             =    tokenizer.encode(a_		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Dict             =    rust_tokenizer.encode(a_		,    add_special_tokens=a_			)
  				self.assertListEqual(a_		,    a_			)
  
  				lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  				lowerCAmelCase_     :							str             =    tokenizer.encode(a_			)
  				lowerCAmelCase_     :							Optional[int]             =    rust_tokenizer.encode(a_			)
  				self.assertListEqual(a_		,    a_			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)		,    ["ah", "\u535A", "\u63A8", "zz"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["hello", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							Optional[int]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hällo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["h\u00E9llo"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				lowerCAmelCase_     :							Dict             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["hallo", "!", "how", "are", "you", "?"]			)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)		,    ["hello"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							Optional[Any]             =    BasicTokenizer(do_lower_case=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[Any]			):
  				lowerCAmelCase_     :							List[Any]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HäLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Dict			):
  				lowerCAmelCase_     :							List[str]             =    BasicTokenizer(do_lower_case=a_		,    strip_accents=a_			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)		,    ["HaLLo", "!", "how", "Are", "yoU", "?"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			List[str]			):
  				lowerCAmelCase_     :							Any             =    BasicTokenizer(do_lower_case=a_		,    never_split=["[UNK]"]			)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)		,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				lowerCAmelCase_     :							int             =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				lowerCAmelCase_     :							Union[str, Any]             =    {}
  				for i, token in enumerate(a_			):
  						lowerCAmelCase_     :							Any             =    i
  				lowerCAmelCase_     :							int             =    WordpieceTokenizer(vocab=a_		,    unk_token="[UNK]"			)
  
  				self.assertListEqual(tokenizer.tokenize(""			)		,    []			)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"			)		,    ["un", "##want", "##ed", "runn", "##ing"]			)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"			)		,    ["[UNK]", "runn", "##ing"]			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Tuple			):
  				self.assertTrue(_is_whitespace(" "			)			)
  				self.assertTrue(_is_whitespace("\t"			)			)
  				self.assertTrue(_is_whitespace("\r"			)			)
  				self.assertTrue(_is_whitespace("\n"			)			)
  				self.assertTrue(_is_whitespace("\u00A0"			)			)
  
  				self.assertFalse(_is_whitespace("A"			)			)
  				self.assertFalse(_is_whitespace("-"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[Any]			):
  				self.assertTrue(_is_control("\u0005"			)			)
  
  				self.assertFalse(_is_control("A"			)			)
  				self.assertFalse(_is_control(" "			)			)
  				self.assertFalse(_is_control("\t"			)			)
  				self.assertFalse(_is_control("\r"			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				self.assertTrue(_is_punctuation("-"			)			)
  				self.assertTrue(_is_punctuation("$"			)			)
  				self.assertTrue(_is_punctuation("`"			)			)
  				self.assertTrue(_is_punctuation("."			)			)
  
  				self.assertFalse(_is_punctuation("A"			)			)
  				self.assertFalse(_is_punctuation(" "			)			)
  
  
  
  
  
  		def      lowerCamelCase (     self							:			str			):
  				lowerCAmelCase_     :							Any             =    self.get_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  				if self.test_rust_tokenizer:
  						lowerCAmelCase_     :							str             =    self.get_rust_tokenizer()
  						self.assertListEqual(
  						    [rust_tokenizer.tokenize(a_			) for t in ["Test", "\xad", "test"]]		,    [["[UNK]"], [], ["[UNK]"]]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				lowerCAmelCase_     :							int             =    self.tokenizer_class.from_pretrained("bert-base-uncased"			)
  
  				lowerCAmelCase_     :							List[Any]             =    tokenizer.encode("sequence builders"		,    add_special_tokens=a_			)
  				lowerCAmelCase_     :							Optional[int]             =    tokenizer.encode("multi-sequence build"		,    add_special_tokens=a_			)
  
  				lowerCAmelCase_     :							Dict             =    tokenizer.build_inputs_with_special_tokens(a_			)
  				lowerCAmelCase_     :							Tuple             =    tokenizer.build_inputs_with_special_tokens(a_		,    a_			)
  
  				assert encoded_sentence == [1_01] + text + [1_02]
  				assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
  
  
  
  
  
  		def      lowerCamelCase (     self							:			Optional[int]			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  
  								lowerCAmelCase_     :							Tuple             =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.encode_plus(
  								    a_		,    return_attention_mask=a_		,    return_token_type_ids=a_		,    return_offsets_mapping=a_		,    add_special_tokens=a_		,    )
  
  								lowerCAmelCase_     :							List[str]             =    tokenizer_r.do_lower_case if hasattr(a_		,    "do_lower_case"			) else False
  								lowerCAmelCase_     :							int             =    (
  								    [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "A"),
  								        ((1, 2), ","),
  								        ((3, 5), "na"),
  								        ((5, 6), "##ï"),
  								        ((6, 8), "##ve"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "Allen"),
  								        ((21, 23), "##NL"),
  								        ((23, 24), "##P"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								    if not do_lower_case
  								    else [
  								        ((0, 0), tokenizer_r.cls_token),
  								        ((0, 1), "a"),
  								        ((1, 2), ","),
  								        ((3, 8), "naive"),
  								        ((9, 15), tokenizer_r.mask_token),
  								        ((16, 21), "allen"),
  								        ((21, 23), "##nl"),
  								        ((23, 24), "##p"),
  								        ((25, 33), "sentence"),
  								        ((33, 34), "."),
  								        ((0, 0), tokenizer_r.sep_token),
  								    ]
  								)
  
  								self.assertEqual(
  								    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
  								self.assertEqual([e[0] for e in expected_results]		,    tokens["offset_mapping"]			)
  
  
  
  
  
  		@slow
  		def      lowerCamelCase (     self							:			int			):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
  								lowerCAmelCase_     :							Tuple             =    self.rust_tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Tuple             =    self.tokenizer_class.from_pretrained(a_		,    **a_			)
  								lowerCAmelCase_     :							Dict             =    [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]
  
  								lowerCAmelCase_     :							Optional[Any]             =    tokenizer_r.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  								lowerCAmelCase_     :							int             =    tokenizer_p.batch_encode_candidates(a_		,    max_length=10		,    return_tensors="np"			)
  
  								lowerCAmelCase_     :							Dict             =    (2, 2, 10)
  
  								self.assertEqual(encoded_sentence_r["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,    a_			)
  
  								self.assertEqual(encoded_sentence_p["input_ids"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["attention_mask"].shape		,    a_			)
  								self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,    a_			)
  