code: 
  import unittest
  
  from transformers.testing_utils import is_torch_available, require_torch, tooslow
  
  from ...generation.test_utils import torch_device
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  				import torch
  
  				from transformers import (
  				    CpmAntConfig,
  				    CpmAntForCausalLM,
  				    CpmAntModel,
  				    CpmAntTokenizer,
  				)
  
  
  
  
  
  @require_torch
  class    _snake_case      :
  
  
  
  
  
  
  
  				def __init__(       self		,     _a		,     _a=2		,     _a=8		,     _a=True		,     _a=False		,     _a=False		,     _a=False		,     _a=False		,     _a=99		,     _a=32		,     _a=3		,     _a=4		,     _a=37		,     _a=32		,     _a=128		,     _a=8		,     _a=8		,     _a=8		,     _a=1.0		,     _a=True		,     ):
  					__magic_name__	:	Optional[Any]			     =  parent
  					__magic_name__	:	int			     =  batch_size
  					__magic_name__	:	Optional[Any]			     =  seq_length
  					__magic_name__	:	List[str]			     =  is_training
  					__magic_name__	:	str			     =  use_token_type_ids
  					__magic_name__	:	List[Any]			     =  use_input_mask
  					__magic_name__	:	Dict			     =  use_labels
  					__magic_name__	:	Optional[int]			     =  use_mc_token_ids
  					__magic_name__	:	Union[str, Any]			     =  vocab_size
  					__magic_name__	:	Dict			     =  hidden_size
  					__magic_name__	:	List[str]			     =  num_hidden_layers
  					__magic_name__	:	Tuple			     =  num_attention_heads
  					__magic_name__	:	List[Any]			     =  intermediate_size
  					__magic_name__	:	Union[str, Any]			     =  num_buckets
  					__magic_name__	:	str			     =  max_distance
  					__magic_name__	:	Union[str, Any]			     =  prompt_length
  					__magic_name__	:	Tuple			     =  prompt_types
  					__magic_name__	:	int			     =  segment_types
  					__magic_name__	:	Any			     =  init_std
  					__magic_name__	:	List[str]			     =  return_dict
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	int			     =  {}
  					__magic_name__	:	Optional[int]			     =  ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size	).type(torch.intaa	)
  					__magic_name__	:	List[Any]			     =  False
  
  					__magic_name__	:	List[str]			     =  self.get_config()
  
  					return (config, input_ids)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					return CpmAntConfig(
  					    vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     dim_ff=self.intermediate_size		,     position_bias_num_buckets=self.num_buckets		,     position_bias_max_distance=self.max_distance		,     prompt_types=self.prompt_types		,     prompt_length=self.prompt_length		,     segment_types=self.segment_types		,     use_cache=lowerCamelCase__		,     init_std=self.init_std		,     return_dict=self.return_dict		,     )
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     *_a	):
  					__magic_name__	:	Optional[Any]			     =  CpmAntModel(config=lowerCamelCase__	)
  					model.to(lowerCamelCase__	)
  					model.eval()
  
  					__magic_name__	:	List[str]			     =  model(**lowerCamelCase__	).last_hidden_state
  
  					self.parent.assertEqual(hidden_states.shape		,     (self.batch_size, self.seq_length, config.hidden_size)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     *_a	):
  					__magic_name__	:	Tuple			     =  CpmAntForCausalLM(lowerCamelCase__	)
  					model.to(lowerCamelCase__	)
  					__magic_name__	:	int			     =  input_ids["input_ids"].to(lowerCamelCase__	)
  					model.eval()
  
  					__magic_name__	:	str			     =  model(**lowerCamelCase__	)
  					self.parent.assertEqual(
  					    model_output.logits.shape		,     (self.batch_size, self.seq_length, config.vocab_size + config.prompt_types * config.prompt_length)		,     )
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Any			     =  self.prepare_config_and_inputs()
  					return config, inputs_dict
  
  
  
  
  
  
  
  @require_torch
  class    _snake_case      (  _a       ,		_a       ,		unittest.TestCase       ):
  				UpperCamelCase__					   =					(CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
  				UpperCamelCase__					   =					(
  				    {'feature-extraction': CpmAntModel, 'text-generation': CpmAntForCausalLM} if is_torch_available() else {}
  				)
  
  				UpperCamelCase__					   =					False
  				UpperCamelCase__					   =					False
  				UpperCamelCase__					   =					False
  				UpperCamelCase__					   =					False
  				UpperCamelCase__					   =					False
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Tuple			     =  CpmAntModelTester(self	)
  					__magic_name__	:	Dict			     =  ConfigTester(self		,     config_class=lowerCamelCase__	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					self.config_tester.create_and_test_config_common_properties()
  					self.config_tester.create_and_test_config_to_json_string()
  					self.config_tester.create_and_test_config_to_json_file()
  					self.config_tester.create_and_test_config_from_and_save_pretrained()
  					self.config_tester.check_config_can_be_init_without_params()
  					self.config_tester.check_config_arguments_init()
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					unittest.skip("CPMAnt doesn't support input_embeds."	)(self.test_inputs_embeds	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					unittest.skip(
  					    "CPMAnt doesn't support retain grad in hidden_states or attentions, because prompt management will peel off the output.hidden_states from graph.\
                   So is attentions. We strongly recommand you use loss to tune model."	)(self.test_retain_grad_hidden_states_attentions	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Optional[int]			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_cpmant_model(lowerCamelCase__		,     lowerCamelCase__	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	List[Any]			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_lm_head_model(lowerCamelCase__		,     lowerCamelCase__	)
  
  
  
  
  
  
  
  @require_torch
  class    _snake_case      (  unittest.TestCase       ):
  
  
  
  
  
  
  
  				@tooslow
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Optional[Any]			     =  "今天天气真好！"
  					__magic_name__	:	Optional[int]			     =  "openbmb/cpm-ant-10b"
  					__magic_name__	:	List[Any]			     =  CpmAntModel.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	Any			     =  CpmAntTokenizer.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	Union[str, Any]			     =  tokenizer(lowerCamelCase__		,     return_tensors="pt"	)
  					__magic_name__	:	List[Any]			     =  model(**lowerCamelCase__	).last_hidden_state
  
  					__magic_name__	:	Optional[int]			     =  torch.tensor(
  					    [[[6.17_08, 5.92_44, 1.08_35], [6.52_07, 6.28_93, -11.33_24], [-1.01_07, -0.05_76, -5.95_77]]]		,     )
  					self.assertTrue(torch.allclose(hidden_states[:, :3, :3]		,     lowerCamelCase__		,     atol=1e-2	)	)
  
  
  
  
  
  
  
  @require_torch
  class    _snake_case      (  unittest.TestCase       ):
  
  
  
  
  
  
  
  				@tooslow
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	List[str]			     =  "今天天气真好！"
  					__magic_name__	:	Any			     =  "openbmb/cpm-ant-10b"
  					__magic_name__	:	List[str]			     =  CpmAntForCausalLM.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	Tuple			     =  CpmAntTokenizer.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	int			     =  tokenizer(lowerCamelCase__		,     return_tensors="pt"	)
  					__magic_name__	:	str			     =  model(**lowerCamelCase__	).logits
  
  					__magic_name__	:	int			     =  torch.tensor(
  					    [[[-6.42_67, -6.40_83, -6.39_58], [-5.88_02, -5.94_47, -5.78_11], [-5.38_96, -5.48_20, -5.42_95]]]		,     )
  					self.assertTrue(torch.allclose(hidden_states[:, :3, :3]		,     lowerCamelCase__		,     atol=1e-2	)	)
  
  
  
  
  
  
  
  				@tooslow
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Any			     =  "openbmb/cpm-ant-10b"
  					__magic_name__	:	Tuple			     =  CpmAntForCausalLM.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	str			     =  CpmAntTokenizer.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	int			     =  "今天天气不错，"
  					__magic_name__	:	Tuple			     =  "今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的"
  					__magic_name__	:	Dict			     =  tokenizer(lowerCamelCase__		,     return_tensors="pt"	)
  					__magic_name__	:	Tuple			     =  model.generate(**lowerCamelCase__	)
  					__magic_name__	:	Union[str, Any]			     =  tokenizer.batch_decode(lowerCamelCase__	)
  					self.assertEqual(lowerCamelCase__		,     lowerCamelCase__	)
  
  
  
  
  
  
  
  				@tooslow
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Any			     =  "openbmb/cpm-ant-10b"
  					__magic_name__	:	Union[str, Any]			     =  CpmAntForCausalLM.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	List[str]			     =  CpmAntTokenizer.from_pretrained(lowerCamelCase__	)
  					__magic_name__	:	Optional[int]			     =  ["今天天气不错，", "新年快乐，万事如意！"]
  					__magic_name__	:	List[str]			     =  [
  					    "今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的",
  					    "新年快乐，万事如意！在这辞旧迎新的美好时刻，我谨代表《农村新技术》杂志社全体同仁，向一直以来关心、支持《农村新技术》杂志发展的各级领导、各界朋友和广大读者致以最诚挚的",
  					]
  					__magic_name__	:	List[Any]			     =  tokenizer(lowerCamelCase__		,     return_tensors="pt"		,     padding=lowerCamelCase__	)
  					__magic_name__	:	List[Any]			     =  model.generate(**lowerCamelCase__	)
  					__magic_name__	:	int			     =  tokenizer.batch_decode(lowerCamelCase__	)
  					self.assertEqual(lowerCamelCase__		,     lowerCamelCase__	)
  
style_context: 
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class    _snake_case      (  snake_case       ,		unittest.TestCase       ):
  			UpperCamelCase__					   =					ElectraTokenizer
  			UpperCamelCase__					   =					ElectraTokenizerFast
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					filter_non_english
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				super().setUp()
  
  				__magic_name__	:	Optional[Any]			     =  [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				__magic_name__	:	Dict			     =  os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["vocab_file"]	)
  				with open(self.vocab_file		,     "w"		,     encoding="utf-8"	) as vocab_writer:
  					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a	):
  				__magic_name__	:	Dict			     =  "UNwant\u00E9d,running"
  				__magic_name__	:	int			     =  "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  self.tokenizer_class(self.vocab_file	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.tokenize("UNwant\u00E9d,running"	)
  				self.assertListEqual(_a		,     ["un", "##want", "##ed", ",", "runn", "##ing"]	)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(_a	)		,     [9, 6, 7, 12, 10, 11]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				if not self.test_rust_tokenizer:
  					return
  
  				__magic_name__	:	str			     =  self.get_tokenizer()
  				__magic_name__	:	Optional[int]			     =  self.get_rust_tokenizer()
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	List[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	str			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	int			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	Union[str, Any]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Tuple			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Union[str, Any]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				# With lower casing
  				__magic_name__	:	Optional[Any]			     =  self.get_tokenizer(do_lower_case=_a	)
  				__magic_name__	:	Union[str, Any]			     =  self.get_rust_tokenizer(do_lower_case=_a	)
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	Optional[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	List[Any]			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Any			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	List[str]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Dict			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Optional[int]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Tuple			     =  BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)		,     ["ah", "\u535A", "\u63A8", "zz"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["hello", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hällo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["h\u00E9llo"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[int]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HäLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HaLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     never_split=["[UNK]"]	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				__magic_name__	:	Optional[int]			     =  {}
  				for i, token in enumerate(_a	):
  					__magic_name__	:	Tuple			     =  i
  				__magic_name__	:	Tuple			     =  WordpieceTokenizer(vocab=_a		,     unk_token="[UNK]"	)
  
  				self.assertListEqual(tokenizer.tokenize(""	)		,     []	)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"	)		,     ["un", "##want", "##ed", "runn", "##ing"]	)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"	)		,     ["[UNK]", "runn", "##ing"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_whitespace(" "	)	)
  				self.assertTrue(_is_whitespace("\t"	)	)
  				self.assertTrue(_is_whitespace("\r"	)	)
  				self.assertTrue(_is_whitespace("\n"	)	)
  				self.assertTrue(_is_whitespace("\u00A0"	)	)
  
  				self.assertFalse(_is_whitespace("A"	)	)
  				self.assertFalse(_is_whitespace("-"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_control("\u0005"	)	)
  
  				self.assertFalse(_is_control("A"	)	)
  				self.assertFalse(_is_control(" "	)	)
  				self.assertFalse(_is_control("\t"	)	)
  				self.assertFalse(_is_control("\r"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_punctuation("-"	)	)
  				self.assertTrue(_is_punctuation("$"	)	)
  				self.assertTrue(_is_punctuation("`"	)	)
  				self.assertTrue(_is_punctuation("."	)	)
  
  				self.assertFalse(_is_punctuation("A"	)	)
  				self.assertFalse(_is_punctuation(" "	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	List[Any]			     =  self.get_tokenizer()
  				__magic_name__	:	List[str]			     =  self.get_rust_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  				self.assertListEqual(
  				    [rust_tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  
  
  
  
  
  
  			@slow
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	int			     =  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)
  
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("sequence builders"		,     add_special_tokens=_a	)
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("multi-sequence build"		,     add_special_tokens=_a	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.build_inputs_with_special_tokens(_a	)
  				__magic_name__	:	Tuple			     =  tokenizer.build_inputs_with_special_tokens(_a		,     _a	)
  
  				assert encoded_sentence == [101] + text + [102]
  				assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	str			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Optional[Any]			     =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  						__magic_name__	:	Any			     =  tokenizer_r.encode_plus(
  						    _a		,     return_attention_mask=_a		,     return_token_type_ids=_a		,     return_offsets_mapping=_a		,     add_special_tokens=_a		,     )
  
  						__magic_name__	:	Any			     =  tokenizer_r.do_lower_case if hasattr(_a		,     "do_lower_case"	) else False
  						__magic_name__	:	int			     =  (
  						    [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "A"),
  						        ((1, 2), ","),
  						        ((3, 5), "na"),
  						        ((5, 6), "##ï"),
  						        ((6, 8), "##ve"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "Allen"),
  						        ((21, 23), "##NL"),
  						        ((23, 24), "##P"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						    if not do_lower_case
  						    else [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "a"),
  						        ((1, 2), ","),
  						        ((3, 8), "naive"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "allen"),
  						        ((21, 23), "##nl"),
  						        ((23, 24), "##p"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						)
  
  						self.assertEqual(
  						    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
  						self.assertEqual([e[0] for e in expected_results]		,     tokens["offset_mapping"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  ["的", "人", "有"]
  				__magic_name__	:	List[str]			     =  "".join(_a	)
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	Optional[int]			     =  True
  						__magic_name__	:	Optional[int]			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	Union[str, Any]			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Tuple			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that each Chinese character is not preceded by "##"
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  
  						__magic_name__	:	Tuple			     =  False
  						__magic_name__	:	Tuple			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	str			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Dict			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	Dict			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that only the first Chinese character is not preceded by "##".
  						__magic_name__	:	Tuple			     =  [
  						    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a	)
  						]
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  