code: 
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class 					snake_case__  ( __A      ,    unittest.TestCase       ):
  	UpperCAmelCase   :							Tuple         = RealmTokenizer
  	UpperCAmelCase   :							Tuple         = RealmTokenizerFast
  	UpperCAmelCase   :							Optional[Any]         = True
  	UpperCAmelCase   :							str         = True
  	UpperCAmelCase   :							int         = filter_non_english
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								super().setUp()
  
  								a_       :   int			=	[
  								    """[UNK]""",
  								    """[CLS]""",
  								    """[SEP]""",
  								    """[PAD]""",
  								    """[MASK]""",
  								    """want""",
  								    """##want""",
  								    """##ed""",
  								    """wa""",
  								    """un""",
  								    """runn""",
  								    """##ing""",
  								    """,""",
  								    """low""",
  								    """lowest""",
  								]
  								a_       :   Any			=	os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES["""vocab_file"""]							)
  								with open(self.vocab_file				,    """w"""				,    encoding="""utf-8"""							) as vocab_writer:
  															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)
  
  
  
  
  
  	def     A				(  self				,    UpperCamelCase_							)		->			Optional[int]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Tuple			=	"""UNwant\u00E9d,running"""
  								a_       :   Tuple			=	"""unwanted, running"""
  								return input_text, output_text
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Dict			=	self.tokenizer_class(self.vocab_file							)
  
  								a_       :   int			=	tokenizer.tokenize("""UNwant\u00E9d,running"""							)
  								self.assertListEqual(UpperCamelCase_				,    ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]							)
  								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)				,    [9, 6, 7, 12, 10, 11]							)
  
  
  
  
  
  	def     A				(  self							)		->			Optional[int]:
  								"""simple docstring"""
  
  
  
  
  
  
  								if not self.test_rust_tokenizer:
  															return
  
  								a_       :   Any			=	self.get_tokenizer()
  								a_       :   Optional[Any]			=	self.get_rust_tokenizer()
  
  								a_       :   Dict			=	"""UNwant\u00E9d,running"""
  
  								a_       :   Any			=	tokenizer.tokenize(UpperCamelCase_							)
  								a_       :   Union[str, Any]			=	rust_tokenizer.tokenize(UpperCamelCase_							)
  								self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_							)
  
  								a_       :   List[str]			=	tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_							)
  								a_       :   List[Any]			=	rust_tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_							)
  								self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_							)
  
  								a_       :   Optional[int]			=	self.get_rust_tokenizer()
  								a_       :   Optional[Any]			=	tokenizer.encode(UpperCamelCase_							)
  								a_       :   str			=	rust_tokenizer.encode(UpperCamelCase_							)
  								self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_							)
  
  								# With lower casing
  								a_       :   int			=	self.get_tokenizer(do_lower_case=UpperCamelCase_							)
  								a_       :   Any			=	self.get_rust_tokenizer(do_lower_case=UpperCamelCase_							)
  
  								a_       :   int			=	"""UNwant\u00E9d,running"""
  
  								a_       :   Optional[Any]			=	tokenizer.tokenize(UpperCamelCase_							)
  								a_       :   Dict			=	rust_tokenizer.tokenize(UpperCamelCase_							)
  								self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_							)
  
  								a_       :   Dict			=	tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_							)
  								a_       :   List[Any]			=	rust_tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_							)
  								self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_							)
  
  								a_       :   Optional[Any]			=	self.get_rust_tokenizer()
  								a_       :   Dict			=	tokenizer.encode(UpperCamelCase_							)
  								a_       :   Optional[int]			=	rust_tokenizer.encode(UpperCamelCase_							)
  								self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_							)
  
  
  
  
  
  	def     A				(  self							)		->			Union[str, Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Tuple			=	BasicTokenizer()
  
  								self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""							)				,    ["""ah""", """\u535A""", """\u63A8""", """zz"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Union[str, Any]			=	BasicTokenizer(do_lower_case=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """							)				,    ["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
  								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)				,    ["""hello"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			Any:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   int			=	BasicTokenizer(do_lower_case=UpperCamelCase_				,    strip_accents=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)				,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""]							)
  								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)				,    ["""h\u00E9llo"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			List[str]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Optional[Any]			=	BasicTokenizer(do_lower_case=UpperCamelCase_				,    strip_accents=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)				,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]							)
  								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)				,    ["""hello"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Optional[int]			=	BasicTokenizer(do_lower_case=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)				,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]							)
  								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)				,    ["""hello"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			Tuple:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Dict			=	BasicTokenizer(do_lower_case=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """							)				,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			Optional[int]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Optional[int]			=	BasicTokenizer(do_lower_case=UpperCamelCase_				,    strip_accents=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)				,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   List[str]			=	BasicTokenizer(do_lower_case=UpperCamelCase_				,    strip_accents=UpperCamelCase_							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)				,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			str:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Dict			=	BasicTokenizer(do_lower_case=UpperCamelCase_				,    never_split=["""[UNK]"""]							)
  
  								self.assertListEqual(
  								    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""							)				,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			Optional[Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Optional[Any]			=	["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]
  
  								a_       :   int			=	{}
  								for i, token in enumerate(UpperCamelCase_							):
  															a_       :   Union[str, Any]			=	i
  								a_       :   Union[str, Any]			=	WordpieceTokenizer(vocab=UpperCamelCase_				,    unk_token="""[UNK]"""							)
  
  								self.assertListEqual(tokenizer.tokenize(""""""							)				,    []							)
  
  								self.assertListEqual(tokenizer.tokenize("""unwanted running"""							)				,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""]							)
  
  								self.assertListEqual(tokenizer.tokenize("""unwantedX running"""							)				,    ["""[UNK]""", """runn""", """##ing"""]							)
  
  
  
  
  
  	def     A				(  self							)		->			Any:
  								"""simple docstring"""
  
  
  
  
  
  
  								self.assertTrue(_is_whitespace(""" """							)							)
  								self.assertTrue(_is_whitespace("""\t"""							)							)
  								self.assertTrue(_is_whitespace("""\r"""							)							)
  								self.assertTrue(_is_whitespace("""\n"""							)							)
  								self.assertTrue(_is_whitespace("""\u00A0"""							)							)
  
  								self.assertFalse(_is_whitespace("""A"""							)							)
  								self.assertFalse(_is_whitespace("""-"""							)							)
  
  
  
  
  
  	def     A				(  self							)		->			Optional[int]:
  								"""simple docstring"""
  
  
  
  
  
  
  								self.assertTrue(_is_control("""\u0005"""							)							)
  
  								self.assertFalse(_is_control("""A"""							)							)
  								self.assertFalse(_is_control(""" """							)							)
  								self.assertFalse(_is_control("""\t"""							)							)
  								self.assertFalse(_is_control("""\r"""							)							)
  
  
  
  
  
  	def     A				(  self							)		->			Union[str, Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								self.assertTrue(_is_punctuation("""-"""							)							)
  								self.assertTrue(_is_punctuation("""$"""							)							)
  								self.assertTrue(_is_punctuation("""`"""							)							)
  								self.assertTrue(_is_punctuation("""."""							)							)
  
  								self.assertFalse(_is_punctuation("""A"""							)							)
  								self.assertFalse(_is_punctuation(""" """							)							)
  
  
  
  
  
  	def     A				(  self							)		->			Any:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   List[str]			=	self.get_tokenizer()
  
  								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  								self.assertListEqual([tokenizer.tokenize(UpperCamelCase_							) for t in ["""Test""", """\xad""", """test"""]]				,    [["""[UNK]"""], [], ["""[UNK]"""]]							)
  
  								if self.test_rust_tokenizer:
  															a_       :   Optional[int]			=	self.get_rust_tokenizer()
  															self.assertListEqual(
  															    [rust_tokenizer.tokenize(UpperCamelCase_							) for t in ["""Test""", """\xad""", """test"""]]				,    [["""[UNK]"""], [], ["""[UNK]"""]]							)
  
  
  
  
  
  	@slow
  	def     A				(  self							)		->			Dict:
  								"""simple docstring"""
  
  
  
  
  
  
  								a_       :   Any			=	self.tokenizer_class.from_pretrained("""bert-base-uncased"""							)
  
  								a_       :   Any			=	tokenizer.encode("""sequence builders"""				,    add_special_tokens=UpperCamelCase_							)
  								a_       :   int			=	tokenizer.encode("""multi-sequence build"""				,    add_special_tokens=UpperCamelCase_							)
  
  								a_       :   Optional[int]			=	tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
  								a_       :   str			=	tokenizer.build_inputs_with_special_tokens(UpperCamelCase_				,    UpperCamelCase_							)
  
  								assert encoded_sentence == [101] + text + [102]
  								assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  	def     A				(  self							)		->			List[Any]:
  								"""simple docstring"""
  
  
  
  
  
  
  								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
  																						a_       :   Dict			=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase_				,    **UpperCamelCase_							)
  
  																						a_       :   List[str]			=	f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
  																						a_       :   Tuple			=	tokenizer_r.encode_plus(
  																						    UpperCamelCase_				,    return_attention_mask=UpperCamelCase_				,    return_token_type_ids=UpperCamelCase_				,    return_offsets_mapping=UpperCamelCase_				,    add_special_tokens=UpperCamelCase_				,    )
  
  																						a_       :   List[str]			=	tokenizer_r.do_lower_case if hasattr(UpperCamelCase_				,    """do_lower_case"""							) else False
  																						a_       :   Union[str, Any]			=	(
  																						    [
  																						        ((0, 0), tokenizer_r.cls_token),
  																						        ((0, 1), """A"""),
  																						        ((1, 2), ""","""),
  																						        ((3, 5), """na"""),
  																						        ((5, 6), """##ï"""),
  																						        ((6, 8), """##ve"""),
  																						        ((9, 15), tokenizer_r.mask_token),
  																						        ((16, 21), """Allen"""),
  																						        ((21, 23), """##NL"""),
  																						        ((23, 24), """##P"""),
  																						        ((25, 33), """sentence"""),
  																						        ((33, 34), """."""),
  																						        ((0, 0), tokenizer_r.sep_token),
  																						    ]
  																						    if not do_lower_case
  																						    else [
  																						        ((0, 0), tokenizer_r.cls_token),
  																						        ((0, 1), """a"""),
  																						        ((1, 2), ""","""),
  																						        ((3, 8), """naive"""),
  																						        ((9, 15), tokenizer_r.mask_token),
  																						        ((16, 21), """allen"""),
  																						        ((21, 23), """##nl"""),
  																						        ((23, 24), """##p"""),
  																						        ((25, 33), """sentence"""),
  																						        ((33, 34), """."""),
  																						        ((0, 0), tokenizer_r.sep_token),
  																						    ]
  																						)
  
  																						self.assertEqual(
  																						    [e[1] for e in expected_results]				,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]							)							)
  																						self.assertEqual([e[0] for e in expected_results]				,    tokens["""offset_mapping"""]							)
  
  
  
  
  
  	@slow
  	def     A				(  self							)		->			int:
  								"""simple docstring"""
  
  
  
  
  
  
  								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
  																						a_       :   List[str]			=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase_				,    **UpperCamelCase_							)
  																						a_       :   Dict			=	self.tokenizer_class.from_pretrained(UpperCamelCase_				,    **UpperCamelCase_							)
  																						a_       :   Optional[Any]			=	[["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]
  
  																						a_       :   Tuple			=	tokenizer_r.batch_encode_candidates(UpperCamelCase_				,    max_length=10				,    return_tensors="""np"""							)
  																						a_       :   Dict			=	tokenizer_p.batch_encode_candidates(UpperCamelCase_				,    max_length=10				,    return_tensors="""np"""							)
  
  																						a_       :   int			=	(2, 2, 10)
  
  																						self.assertEqual(encoded_sentence_r["""input_ids"""].shape				,    UpperCamelCase_							)
  																						self.assertEqual(encoded_sentence_r["""attention_mask"""].shape				,    UpperCamelCase_							)
  																						self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape				,    UpperCamelCase_							)
  
  																						self.assertEqual(encoded_sentence_p["""input_ids"""].shape				,    UpperCamelCase_							)
  																						self.assertEqual(encoded_sentence_p["""attention_mask"""].shape				,    UpperCamelCase_							)
  																						self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape				,    UpperCamelCase_							)
  
  
  
style_context: 
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_sentencepiece_available,
      is_tf_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  SCREAMING_SNAKE_CASE : Optional[Any]          =    {
      "configuration_camembert": ["CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "CamembertConfig", "CamembertOnnxConfig"],
  }
  
  try:
  	if not is_sentencepiece_available():
  		raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  	pass
  else:
  	SCREAMING_SNAKE_CASE : Optional[int]          =    ["CamembertTokenizer"]
  
  try:
  	if not is_tokenizers_available():
  		raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  	pass
  else:
  	SCREAMING_SNAKE_CASE : Any          =    ["CamembertTokenizerFast"]
  
  try:
  	if not is_torch_available():
  		raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  	pass
  else:
  	SCREAMING_SNAKE_CASE : Union[str, Any]          =    [
  	    "CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
  	    "CamembertForCausalLM",
  	    "CamembertForMaskedLM",
  	    "CamembertForMultipleChoice",
  	    "CamembertForQuestionAnswering",
  	    "CamembertForSequenceClassification",
  	    "CamembertForTokenClassification",
  	    "CamembertModel",
  	    "CamembertPreTrainedModel",
  	]
  
  try:
  	if not is_tf_available():
  		raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  	pass
  else:
  	SCREAMING_SNAKE_CASE : List[Any]          =    [
  	    "TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
  	    "TFCamembertForCausalLM",
  	    "TFCamembertForMaskedLM",
  	    "TFCamembertForMultipleChoice",
  	    "TFCamembertForQuestionAnswering",
  	    "TFCamembertForSequenceClassification",
  	    "TFCamembertForTokenClassification",
  	    "TFCamembertModel",
  	    "TFCamembertPreTrainedModel",
  	]
  
  
  if TYPE_CHECKING:
  	from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig, CamembertOnnxConfig
  
  	try:
  		if not is_sentencepiece_available():
  			raise OptionalDependencyNotAvailable()
  	except OptionalDependencyNotAvailable:
  		pass
  	else:
  		from .tokenization_camembert import CamembertTokenizer
  
  	try:
  		if not is_tokenizers_available():
  			raise OptionalDependencyNotAvailable()
  	except OptionalDependencyNotAvailable:
  		pass
  	else:
  		from .tokenization_camembert_fast import CamembertTokenizerFast
  
  	try:
  		if not is_torch_available():
  			raise OptionalDependencyNotAvailable()
  	except OptionalDependencyNotAvailable:
  		pass
  	else:
  		from .modeling_camembert import (
  		    CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
  		    CamembertForCausalLM,
  		    CamembertForMaskedLM,
  		    CamembertForMultipleChoice,
  		    CamembertForQuestionAnswering,
  		    CamembertForSequenceClassification,
  		    CamembertForTokenClassification,
  		    CamembertModel,
  		    CamembertPreTrainedModel,
  		)
  
  	try:
  		if not is_tf_available():
  			raise OptionalDependencyNotAvailable()
  	except OptionalDependencyNotAvailable:
  		pass
  	else:
  		from .modeling_tf_camembert import (
  		    TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
  		    TFCamembertForCausalLM,
  		    TFCamembertForMaskedLM,
  		    TFCamembertForMultipleChoice,
  		    TFCamembertForQuestionAnswering,
  		    TFCamembertForSequenceClassification,
  		    TFCamembertForTokenClassification,
  		    TFCamembertModel,
  		    TFCamembertPreTrainedModel,
  		)
  
  else:
  	import sys
  
  	SCREAMING_SNAKE_CASE : Any          =    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
  
  
  