code: 
  import argparse
  import os
  import pathlib
  
  import fairseq
  import torch
  from fairseq.modules import TransformerSentenceEncoderLayer
  from packaging import version
  
  from transformers import (
      DataaVecTextConfig,
      DataaVecTextForMaskedLM,
      DataaVecTextForSequenceClassification,
      DataaVecTextModel,
  )
  from transformers.models.bert.modeling_bert import (
      BertIntermediate,
      BertLayer,
      BertOutput,
      BertSelfAttention,
      BertSelfOutput,
  )
  
  # IMPORTANT: In order for this script to run, please make sure to download the dictionary: `dict.txt` from wget https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz
  # File copied from https://github.com/pytorch/fairseq/blob/main/examples/data2vec/models/data2vec_text.py
  from transformers.utils import logging
  
  
  if version.parse(fairseq.__version__) < version.parse("0.9.0"):
  				raise Exception("requires fairseq >= 0.9.0")
  
  
  logging.set_verbosity_info()
  snake_case				:	Dict							   =						logging.get_logger(__name__)
  
  snake_case				:	Tuple							   =						"Hello world! cécé herlolip"
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						str   ,     _snake_case   :						str   ,     _snake_case   :						bool     )       ->	Any:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__     ,					__magic_name__	:	List[str]			     =  os.path.split(_snake_case     )
  	__magic_name__	:	List[str]			     =  DataaVecTextModel.from_pretrained(
  	    _snake_case   ,     checkpoint_file=_snake_case     )
  	dataavec.eval()  # disable dropout
  	__magic_name__	:	List[str]			     =  dataavec.models[0]
  	__magic_name__	:	Tuple			     =  dataavec_model.encoder.sentence_encoder
  	__magic_name__	:	Any			     =  DataaVecTextConfig(
  	    vocab_size=dataavec_sent_encoder.embed_tokens.num_embeddings   ,     hidden_size=dataavec_model.args.encoder_embed_dim   ,     num_hidden_layers=dataavec_model.args.encoder_layers   ,     num_attention_heads=dataavec_model.args.encoder_attention_heads   ,     intermediate_size=dataavec_model.args.encoder_ffn_embed_dim   ,     max_position_embeddings=514   ,     type_vocab_size=1   ,     layer_norm_eps=1E-5   ,     )
  	if classification_head:
  		__magic_name__	:	Optional[Any]			     =  dataavec.model.classification_heads["mnli"].out_proj.weight.shape[0]
  	print("Our BERT config:"   ,     _snake_case     )
  
  	__magic_name__	:	Dict			     =  DataaVecTextForSequenceClassification(_snake_case     ) if classification_head else DataaVecTextForMaskedLM(_snake_case     )
  	model.eval()
  
  	# Now let's copy all the weights.
  	# Embeddings
  	__magic_name__	:	Optional[int]			     =  dataavec_sent_encoder.embed_tokens.weight
  	__magic_name__	:	List[str]			     =  dataavec_sent_encoder.embed_positions.weight
  	__magic_name__	:	str			     =  torch.zeros_like(
  	    model.dataavec_text.embeddings.token_type_embeddings.weight     )  # just zero them out b/c data2vec doesn't use them.
  	__magic_name__	:	List[Any]			     =  dataavec_sent_encoder.layernorm_embedding.weight
  	__magic_name__	:	Optional[Any]			     =  dataavec_sent_encoder.layernorm_embedding.bias
  
  	for i in range(config.num_hidden_layers     ):
  		# Encoder: start of layer
  		__magic_name__	:	BertLayer			     =  model.dataavec_text.encoder.layer[i]
  		__magic_name__	:	TransformerSentenceEncoderLayer			     =  dataavec_sent_encoder.layers[i]
  
  		# self attention
  		__magic_name__	:	BertSelfAttention			     =  layer.attention.self
  		assert dataavec_layer.self_attn.k_proj.weight.data.shape == torch.Size(
  		    (config.hidden_size, config.hidden_size)     ), (
  		    "Shape for data2vec_layer.self_attn.k_proj.weight.data should be"
  		    F''' {torch.Size((config.hidden_size, config.hidden_size)     )}'''
  		)
  		assert dataavec_layer.self_attn.q_proj.weight.data.shape == torch.Size(
  		    (config.hidden_size, config.hidden_size)     ), (
  		    "Shape for data2vec_layer.self_attn.q_proj.weight.data should be"
  		    F''' {torch.Size((config.hidden_size, config.hidden_size)     )}'''
  		)
  		assert dataavec_layer.self_attn.v_proj.weight.data.shape == torch.Size(
  		    (config.hidden_size, config.hidden_size)     ), (
  		    "Shape for data2vec_layer.self_attn.v_proj.weight.data should be"
  		    F''' {torch.Size((config.hidden_size, config.hidden_size)     )}'''
  		)
  
  		__magic_name__	:	int			     =  dataavec_layer.self_attn.q_proj.weight
  		__magic_name__	:	Any			     =  dataavec_layer.self_attn.q_proj.bias
  		__magic_name__	:	str			     =  dataavec_layer.self_attn.k_proj.weight
  		__magic_name__	:	str			     =  dataavec_layer.self_attn.k_proj.bias
  		__magic_name__	:	List[str]			     =  dataavec_layer.self_attn.v_proj.weight
  		__magic_name__	:	Union[str, Any]			     =  dataavec_layer.self_attn.v_proj.bias
  
  		# self-attention output
  		__magic_name__	:	BertSelfOutput			     =  layer.attention.output
  		assert (
  		    self_output.dense.weight.shape == dataavec_layer.self_attn.out_proj.weight.shape
  		), F'''Shape for self_output.dense.weight should be {dataavec_layer.self_attn.out_proj.weight.shape}'''
  		__magic_name__	:	Union[str, Any]			     =  dataavec_layer.self_attn.out_proj.weight
  		__magic_name__	:	Any			     =  dataavec_layer.self_attn.out_proj.bias
  		__magic_name__	:	Any			     =  dataavec_layer.self_attn_layer_norm.weight
  		__magic_name__	:	Optional[int]			     =  dataavec_layer.self_attn_layer_norm.bias
  
  		# intermediate
  		__magic_name__	:	BertIntermediate			     =  layer.intermediate
  		assert (
  		    intermediate.dense.weight.shape == dataavec_layer.fca.weight.shape
  		), F'''Shape for intermediate.dense.weight should be {dataavec_layer.fca.weight.shape}'''
  		__magic_name__	:	Tuple			     =  dataavec_layer.fca.weight
  		__magic_name__	:	str			     =  dataavec_layer.fca.bias
  
  		# output
  		__magic_name__	:	BertOutput			     =  layer.output
  		assert (
  		    bert_output.dense.weight.shape == dataavec_layer.fca.weight.shape
  		), F'''Shape for bert_output.dense.weight should be {dataavec_layer.fca.weight.shape}'''
  		__magic_name__	:	Optional[int]			     =  dataavec_layer.fca.weight
  		__magic_name__	:	Optional[int]			     =  dataavec_layer.fca.bias
  		__magic_name__	:	Union[str, Any]			     =  dataavec_layer.final_layer_norm.weight
  		__magic_name__	:	Dict			     =  dataavec_layer.final_layer_norm.bias
  		# end of layer
  
  	if classification_head:
  		__magic_name__	:	List[Any]			     =  dataavec.model.classification_heads["mnli"].dense.weight
  		__magic_name__	:	Dict			     =  dataavec.model.classification_heads["mnli"].dense.bias
  		__magic_name__	:	Optional[Any]			     =  dataavec.model.classification_heads["mnli"].out_proj.weight
  		__magic_name__	:	Tuple			     =  dataavec.model.classification_heads["mnli"].out_proj.bias
  	else:
  		# LM Head
  		__magic_name__	:	List[Any]			     =  dataavec_model.encoder.lm_head.dense.weight
  		__magic_name__	:	List[str]			     =  dataavec_model.encoder.lm_head.dense.bias
  		__magic_name__	:	Union[str, Any]			     =  dataavec_model.encoder.lm_head.layer_norm.weight
  		__magic_name__	:	Tuple			     =  dataavec_model.encoder.lm_head.layer_norm.bias
  		__magic_name__	:	Optional[Any]			     =  dataavec_model.encoder.lm_head.weight
  		__magic_name__	:	Union[str, Any]			     =  dataavec_model.encoder.lm_head.bias
  
  	# Let's check that we get the same results.
  	__magic_name__	:	torch.Tensor			     =  dataavec.encode(_snake_case     ).unsqueeze(0     )  # batch of size 1
  
  	__magic_name__	:	Union[str, Any]			     =  model(_snake_case     )[0]
  	if classification_head:
  		__magic_name__	:	Dict			     =  dataavec.model.classification_heads["mnli"](dataavec.extract_features(_snake_case     )     )
  	else:
  		__magic_name__	:	List[Any]			     =  dataavec_model(_snake_case     )[0]
  	print(our_output.shape   ,     their_output.shape     )
  	__magic_name__	:	List[Any]			     =  torch.max(torch.abs(our_output - their_output     )     ).item()
  	print(F'''max_absolute_diff = {max_absolute_diff}'''     )  # ~ 1e-7
  	__magic_name__	:	Any			     =  torch.allclose(_snake_case   ,     _snake_case   ,     atol=1E-3     )
  	print("Do both models output the same tensors?"   ,     "🔥" if success else "💩"     )
  	if not success:
  		raise Exception("Something went wRoNg"     )
  
  	pathlib.Path(_snake_case     ).mkdir(parents=_snake_case   ,     exist_ok=_snake_case     )
  	print(F'''Saving model to {pytorch_dump_folder_path}'''     )
  	model.save_pretrained(_snake_case     )
  
  
  if __name__ == "__main__":
  				snake_case				:	List[str]							   =						argparse.ArgumentParser()
  				# Required parameters
  				parser.add_argument(
  				    "--checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
  				)
  				parser.add_argument(
  				    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
  				)
  				parser.add_argument(
  				    "--classification_head", action="store_true", help="Whether to convert a final classification head."
  				)
  				snake_case				:	Dict							   =						parser.parse_args()
  				convert_dataavec_checkpoint_to_pytorch(
  				    args.checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
  				)
  
style_context: 
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class    _snake_case      (  snake_case       ,		unittest.TestCase       ):
  			UpperCamelCase__					   =					ElectraTokenizer
  			UpperCamelCase__					   =					ElectraTokenizerFast
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					filter_non_english
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				super().setUp()
  
  				__magic_name__	:	Optional[Any]			     =  [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				__magic_name__	:	Dict			     =  os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["vocab_file"]	)
  				with open(self.vocab_file		,     "w"		,     encoding="utf-8"	) as vocab_writer:
  					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a	):
  				__magic_name__	:	Dict			     =  "UNwant\u00E9d,running"
  				__magic_name__	:	int			     =  "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  self.tokenizer_class(self.vocab_file	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.tokenize("UNwant\u00E9d,running"	)
  				self.assertListEqual(_a		,     ["un", "##want", "##ed", ",", "runn", "##ing"]	)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(_a	)		,     [9, 6, 7, 12, 10, 11]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				if not self.test_rust_tokenizer:
  					return
  
  				__magic_name__	:	str			     =  self.get_tokenizer()
  				__magic_name__	:	Optional[int]			     =  self.get_rust_tokenizer()
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	List[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	str			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	int			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	Union[str, Any]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Tuple			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Union[str, Any]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				# With lower casing
  				__magic_name__	:	Optional[Any]			     =  self.get_tokenizer(do_lower_case=_a	)
  				__magic_name__	:	Union[str, Any]			     =  self.get_rust_tokenizer(do_lower_case=_a	)
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	Optional[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	List[Any]			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Any			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	List[str]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Dict			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Optional[int]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Tuple			     =  BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)		,     ["ah", "\u535A", "\u63A8", "zz"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["hello", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hällo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["h\u00E9llo"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[int]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HäLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HaLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     never_split=["[UNK]"]	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				__magic_name__	:	Optional[int]			     =  {}
  				for i, token in enumerate(_a	):
  					__magic_name__	:	Tuple			     =  i
  				__magic_name__	:	Tuple			     =  WordpieceTokenizer(vocab=_a		,     unk_token="[UNK]"	)
  
  				self.assertListEqual(tokenizer.tokenize(""	)		,     []	)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"	)		,     ["un", "##want", "##ed", "runn", "##ing"]	)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"	)		,     ["[UNK]", "runn", "##ing"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_whitespace(" "	)	)
  				self.assertTrue(_is_whitespace("\t"	)	)
  				self.assertTrue(_is_whitespace("\r"	)	)
  				self.assertTrue(_is_whitespace("\n"	)	)
  				self.assertTrue(_is_whitespace("\u00A0"	)	)
  
  				self.assertFalse(_is_whitespace("A"	)	)
  				self.assertFalse(_is_whitespace("-"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_control("\u0005"	)	)
  
  				self.assertFalse(_is_control("A"	)	)
  				self.assertFalse(_is_control(" "	)	)
  				self.assertFalse(_is_control("\t"	)	)
  				self.assertFalse(_is_control("\r"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_punctuation("-"	)	)
  				self.assertTrue(_is_punctuation("$"	)	)
  				self.assertTrue(_is_punctuation("`"	)	)
  				self.assertTrue(_is_punctuation("."	)	)
  
  				self.assertFalse(_is_punctuation("A"	)	)
  				self.assertFalse(_is_punctuation(" "	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	List[Any]			     =  self.get_tokenizer()
  				__magic_name__	:	List[str]			     =  self.get_rust_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  				self.assertListEqual(
  				    [rust_tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  
  
  
  
  
  
  			@slow
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	int			     =  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)
  
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("sequence builders"		,     add_special_tokens=_a	)
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("multi-sequence build"		,     add_special_tokens=_a	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.build_inputs_with_special_tokens(_a	)
  				__magic_name__	:	Tuple			     =  tokenizer.build_inputs_with_special_tokens(_a		,     _a	)
  
  				assert encoded_sentence == [101] + text + [102]
  				assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	str			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Optional[Any]			     =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  						__magic_name__	:	Any			     =  tokenizer_r.encode_plus(
  						    _a		,     return_attention_mask=_a		,     return_token_type_ids=_a		,     return_offsets_mapping=_a		,     add_special_tokens=_a		,     )
  
  						__magic_name__	:	Any			     =  tokenizer_r.do_lower_case if hasattr(_a		,     "do_lower_case"	) else False
  						__magic_name__	:	int			     =  (
  						    [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "A"),
  						        ((1, 2), ","),
  						        ((3, 5), "na"),
  						        ((5, 6), "##ï"),
  						        ((6, 8), "##ve"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "Allen"),
  						        ((21, 23), "##NL"),
  						        ((23, 24), "##P"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						    if not do_lower_case
  						    else [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "a"),
  						        ((1, 2), ","),
  						        ((3, 8), "naive"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "allen"),
  						        ((21, 23), "##nl"),
  						        ((23, 24), "##p"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						)
  
  						self.assertEqual(
  						    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
  						self.assertEqual([e[0] for e in expected_results]		,     tokens["offset_mapping"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  ["的", "人", "有"]
  				__magic_name__	:	List[str]			     =  "".join(_a	)
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	Optional[int]			     =  True
  						__magic_name__	:	Optional[int]			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	Union[str, Any]			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Tuple			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that each Chinese character is not preceded by "##"
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  
  						__magic_name__	:	Tuple			     =  False
  						__magic_name__	:	Tuple			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	str			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Dict			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	Dict			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that only the first Chinese character is not preceded by "##".
  						__magic_name__	:	Tuple			     =  [
  						    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a	)
  						]
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  