code: 
  import unittest
  
  from transformers import ElectraConfig, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  				import torch
  
  				from transformers import (
  				    MODEL_FOR_PRETRAINING_MAPPING,
  				    ElectraForCausalLM,
  				    ElectraForMaskedLM,
  				    ElectraForMultipleChoice,
  				    ElectraForPreTraining,
  				    ElectraForQuestionAnswering,
  				    ElectraForSequenceClassification,
  				    ElectraForTokenClassification,
  				    ElectraModel,
  				)
  				from transformers.models.electra.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  class    _snake_case      :
  
  
  
  
  
  
  
  				def __init__(       self		,     _a		,     _a=13		,     _a=7		,     _a=True		,     _a=True		,     _a=True		,     _a=True		,     _a=99		,     _a=32		,     _a=5		,     _a=4		,     _a=37		,     _a="gelu"		,     _a=0.1		,     _a=0.1		,     _a=512		,     _a=16		,     _a=2		,     _a=0.02		,     _a=3		,     _a=4		,     _a=None		,     ):
  					__magic_name__	:	Dict			     =  parent
  					__magic_name__	:	List[Any]			     =  batch_size
  					__magic_name__	:	int			     =  seq_length
  					__magic_name__	:	int			     =  is_training
  					__magic_name__	:	str			     =  use_input_mask
  					__magic_name__	:	List[str]			     =  use_token_type_ids
  					__magic_name__	:	Any			     =  use_labels
  					__magic_name__	:	List[Any]			     =  vocab_size
  					__magic_name__	:	List[str]			     =  hidden_size
  					__magic_name__	:	int			     =  num_hidden_layers
  					__magic_name__	:	Dict			     =  num_attention_heads
  					__magic_name__	:	Optional[Any]			     =  intermediate_size
  					__magic_name__	:	Dict			     =  hidden_act
  					__magic_name__	:	Any			     =  hidden_dropout_prob
  					__magic_name__	:	Dict			     =  attention_probs_dropout_prob
  					__magic_name__	:	Optional[int]			     =  max_position_embeddings
  					__magic_name__	:	Optional[Any]			     =  type_vocab_size
  					__magic_name__	:	List[Any]			     =  type_sequence_label_size
  					__magic_name__	:	List[str]			     =  initializer_range
  					__magic_name__	:	Optional[Any]			     =  num_labels
  					__magic_name__	:	str			     =  num_choices
  					__magic_name__	:	List[str]			     =  scope
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Tuple			     =  ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size	)
  
  					__magic_name__	:	List[Any]			     =  None
  					if self.use_input_mask:
  						__magic_name__	:	Tuple			     =  random_attention_mask([self.batch_size, self.seq_length]	)
  
  					__magic_name__	:	List[str]			     =  None
  					if self.use_token_type_ids:
  						__magic_name__	:	List[Any]			     =  ids_tensor([self.batch_size, self.seq_length]		,     self.type_vocab_size	)
  
  					__magic_name__	:	Tuple			     =  None
  					__magic_name__	:	str			     =  None
  					__magic_name__	:	Dict			     =  None
  					if self.use_labels:
  						__magic_name__	:	List[str]			     =  ids_tensor([self.batch_size]		,     self.type_sequence_label_size	)
  						__magic_name__	:	int			     =  ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels	)
  						__magic_name__	:	List[Any]			     =  ids_tensor([self.batch_size]		,     self.num_choices	)
  						__magic_name__	:	List[Any]			     =  ids_tensor([self.batch_size, self.seq_length]		,     1	)
  
  					__magic_name__	:	Any			     =  self.get_config()
  
  					return (
  					    config,
  					    input_ids,
  					    token_type_ids,
  					    input_mask,
  					    sequence_labels,
  					    token_labels,
  					    choice_labels,
  					    fake_token_labels,
  					)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					return ElectraConfig(
  					    vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     intermediate_size=self.intermediate_size		,     hidden_act=self.hidden_act		,     hidden_dropout_prob=self.hidden_dropout_prob		,     attention_probs_dropout_prob=self.attention_probs_dropout_prob		,     max_position_embeddings=self.max_position_embeddings		,     type_vocab_size=self.type_vocab_size		,     is_decoder=_a		,     initializer_range=self.initializer_range		,     )
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					(
  					    __magic_name__					
  					)	:	Any			     =  self.prepare_config_and_inputs()
  
  					__magic_name__	:	List[str]			     =  True
  					__magic_name__	:	Any			     =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
  					__magic_name__	:	Any			     =  ids_tensor([self.batch_size, self.seq_length]		,     vocab_size=2	)
  
  					return (
  					    config,
  					    input_ids,
  					    token_type_ids,
  					    input_mask,
  					    sequence_labels,
  					    token_labels,
  					    choice_labels,
  					    encoder_hidden_states,
  					    encoder_attention_mask,
  					)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	Any			     =  ElectraModel(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	Tuple			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a	)
  					__magic_name__	:	Optional[Any]			     =  model(_a		,     token_type_ids=_a	)
  					__magic_name__	:	List[Any]			     =  model(_a	)
  					self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	Tuple			     =  True
  					__magic_name__	:	List[str]			     =  ElectraModel(_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	Union[str, Any]			     =  model(
  					    _a		,     attention_mask=_a		,     token_type_ids=_a		,     encoder_hidden_states=_a		,     encoder_attention_mask=_a		,     )
  					__magic_name__	:	Optional[int]			     =  model(
  					    _a		,     attention_mask=_a		,     token_type_ids=_a		,     encoder_hidden_states=_a		,     )
  					__magic_name__	:	Optional[Any]			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a	)
  					self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	Any			     =  ElectraForMaskedLM(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	List[Any]			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a		,     labels=_a	)
  					self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	List[str]			     =  ElectraForCausalLM(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	Tuple			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a		,     labels=_a	)
  					self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	int			     =  self.num_labels
  					__magic_name__	:	List[str]			     =  ElectraForTokenClassification(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	List[str]			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a		,     labels=_a	)
  					self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.num_labels)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	str			     =  self.num_labels
  					__magic_name__	:	List[Any]			     =  ElectraForPreTraining(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	List[Any]			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a		,     labels=_a	)
  					self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	List[str]			     =  self.num_labels
  					__magic_name__	:	Dict			     =  ElectraForSequenceClassification(_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	str			     =  model(_a		,     attention_mask=_a		,     token_type_ids=_a		,     labels=_a	)
  					self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_labels)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	Dict			     =  ElectraForQuestionAnswering(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	Tuple			     =  model(
  					    _a		,     attention_mask=_a		,     token_type_ids=_a		,     start_positions=_a		,     end_positions=_a		,     )
  					self.parent.assertEqual(result.start_logits.shape		,     (self.batch_size, self.seq_length)	)
  					self.parent.assertEqual(result.end_logits.shape		,     (self.batch_size, self.seq_length)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     _a		,     ):
  					__magic_name__	:	str			     =  self.num_choices
  					__magic_name__	:	Union[str, Any]			     =  ElectraForMultipleChoice(config=_a	)
  					model.to(_a	)
  					model.eval()
  					__magic_name__	:	str			     =  input_ids.unsqueeze(1	).expand(-1		,     self.num_choices		,     -1	).contiguous()
  					__magic_name__	:	int			     =  token_type_ids.unsqueeze(1	).expand(-1		,     self.num_choices		,     -1	).contiguous()
  					__magic_name__	:	Dict			     =  input_mask.unsqueeze(1	).expand(-1		,     self.num_choices		,     -1	).contiguous()
  					__magic_name__	:	Union[str, Any]			     =  model(
  					    _a		,     attention_mask=_a		,     token_type_ids=_a		,     labels=_a		,     )
  					self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_choices)	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Tuple			     =  self.prepare_config_and_inputs()
  					(
  					    __magic_name__					
  					)	:	List[str]			     =  config_and_inputs
  					__magic_name__	:	Optional[int]			     =  {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
  					return config, inputs_dict
  
  
  
  
  
  
  
  @require_torch
  class    _snake_case      (  snake_case       ,		snake_case       ,		unittest.TestCase       ):
  				UpperCamelCase__					   =					(
  				    (
  				        ElectraModel,
  				        ElectraForPreTraining,
  				        ElectraForMaskedLM,
  				        ElectraForCausalLM,
  				        ElectraForMultipleChoice,
  				        ElectraForTokenClassification,
  				        ElectraForSequenceClassification,
  				        ElectraForQuestionAnswering,
  				    )
  				    if is_torch_available()
  				    else ()
  				)
  				UpperCamelCase__					   =					(
  				    {
  				        'feature-extraction': ElectraModel,
  				        'fill-mask': ElectraForMaskedLM,
  				        'question-answering': ElectraForQuestionAnswering,
  				        'text-classification': ElectraForSequenceClassification,
  				        'text-generation': ElectraForCausalLM,
  				        'token-classification': ElectraForTokenClassification,
  				        'zero-shot': ElectraForSequenceClassification,
  				    }
  				    if is_torch_available()
  				    else {}
  				)
  				UpperCamelCase__					   =					True
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self		,     _a		,     _a		,     _a=False	):
  					__magic_name__	:	Union[str, Any]			     =  super()._prepare_for_class(_a		,     _a		,     return_labels=_a	)
  
  					if return_labels:
  						if model_class in get_values(_a	):
  							__magic_name__	:	Optional[int]			     =  torch.zeros(
  							    (self.model_tester.batch_size, self.model_tester.seq_length)		,     dtype=torch.long		,     device=_a	)
  					return inputs_dict
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	List[Any]			     =  ElectraModelTester(self	)
  					__magic_name__	:	Optional[Any]			     =  ConfigTester(self		,     config_class=_a		,     hidden_size=37	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					self.config_tester.run_common_tests()
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Optional[int]			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_model(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Tuple			     =  self.model_tester.prepare_config_and_inputs_for_decoder()
  					self.model_tester.create_and_check_electra_model_as_decoder(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Optional[int]			     =  self.model_tester.prepare_config_and_inputs()
  					for type in ["absolute", "relative_key", "relative_key_query"]:
  						__magic_name__	:	Tuple			     =  type
  						self.model_tester.create_and_check_electra_model(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Any			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_for_masked_lm(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Dict			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_for_token_classification(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	List[Any]			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_for_pretraining(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Dict			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_for_sequence_classification(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Any			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_for_question_answering(*_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	int			     =  self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_electra_for_multiple_choice(*_a	)
  
  
  
  
  
  
  
  				@slow
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					for model_name in ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  						__magic_name__	:	Optional[int]			     =  ElectraModel.from_pretrained(_a	)
  						self.assertIsNotNone(_a	)
  
  
  
  
  
  
  
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Tuple			     =  self.model_tester.prepare_config_and_inputs_for_decoder()
  					self.model_tester.create_and_check_electra_for_causal_lm(*_a	)
  
  
  
  
  
  
  
  @require_torch
  class    _snake_case      (  unittest.TestCase       ):
  
  
  
  
  
  
  
  				@slow
  				def     SCREAMING_SNAKE_CASE					(       self	):
  					__magic_name__	:	Dict			     =  ElectraModel.from_pretrained("google/electra-small-discriminator"	)
  					__magic_name__	:	List[Any]			     =  torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]	)
  					__magic_name__	:	int			     =  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
  					__magic_name__	:	str			     =  model(_a		,     attention_mask=_a	)[0]
  					__magic_name__	:	Dict			     =  torch.Size((1, 11, 256)	)
  					self.assertEqual(output.shape		,     _a	)
  					__magic_name__	:	Tuple			     =  torch.tensor(
  					    [[[0.44_71, 0.68_21, -0.32_65], [0.46_27, 0.52_55, -0.36_68], [0.45_32, 0.33_13, -0.43_44]]]	)
  
  					self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,     _a		,     atol=1e-4	)	)
  
style_context: 
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class    _snake_case      (  snake_case       ,		unittest.TestCase       ):
  			UpperCamelCase__					   =					ElectraTokenizer
  			UpperCamelCase__					   =					ElectraTokenizerFast
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					filter_non_english
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				super().setUp()
  
  				__magic_name__	:	Optional[Any]			     =  [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				__magic_name__	:	Dict			     =  os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["vocab_file"]	)
  				with open(self.vocab_file		,     "w"		,     encoding="utf-8"	) as vocab_writer:
  					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a	):
  				__magic_name__	:	Dict			     =  "UNwant\u00E9d,running"
  				__magic_name__	:	int			     =  "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  self.tokenizer_class(self.vocab_file	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.tokenize("UNwant\u00E9d,running"	)
  				self.assertListEqual(_a		,     ["un", "##want", "##ed", ",", "runn", "##ing"]	)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(_a	)		,     [9, 6, 7, 12, 10, 11]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				if not self.test_rust_tokenizer:
  					return
  
  				__magic_name__	:	str			     =  self.get_tokenizer()
  				__magic_name__	:	Optional[int]			     =  self.get_rust_tokenizer()
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	List[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	str			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	int			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	Union[str, Any]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Tuple			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Union[str, Any]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				# With lower casing
  				__magic_name__	:	Optional[Any]			     =  self.get_tokenizer(do_lower_case=_a	)
  				__magic_name__	:	Union[str, Any]			     =  self.get_rust_tokenizer(do_lower_case=_a	)
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	Optional[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	List[Any]			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Any			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	List[str]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Dict			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Optional[int]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Tuple			     =  BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)		,     ["ah", "\u535A", "\u63A8", "zz"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["hello", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hällo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["h\u00E9llo"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[int]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HäLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HaLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     never_split=["[UNK]"]	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				__magic_name__	:	Optional[int]			     =  {}
  				for i, token in enumerate(_a	):
  					__magic_name__	:	Tuple			     =  i
  				__magic_name__	:	Tuple			     =  WordpieceTokenizer(vocab=_a		,     unk_token="[UNK]"	)
  
  				self.assertListEqual(tokenizer.tokenize(""	)		,     []	)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"	)		,     ["un", "##want", "##ed", "runn", "##ing"]	)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"	)		,     ["[UNK]", "runn", "##ing"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_whitespace(" "	)	)
  				self.assertTrue(_is_whitespace("\t"	)	)
  				self.assertTrue(_is_whitespace("\r"	)	)
  				self.assertTrue(_is_whitespace("\n"	)	)
  				self.assertTrue(_is_whitespace("\u00A0"	)	)
  
  				self.assertFalse(_is_whitespace("A"	)	)
  				self.assertFalse(_is_whitespace("-"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_control("\u0005"	)	)
  
  				self.assertFalse(_is_control("A"	)	)
  				self.assertFalse(_is_control(" "	)	)
  				self.assertFalse(_is_control("\t"	)	)
  				self.assertFalse(_is_control("\r"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_punctuation("-"	)	)
  				self.assertTrue(_is_punctuation("$"	)	)
  				self.assertTrue(_is_punctuation("`"	)	)
  				self.assertTrue(_is_punctuation("."	)	)
  
  				self.assertFalse(_is_punctuation("A"	)	)
  				self.assertFalse(_is_punctuation(" "	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	List[Any]			     =  self.get_tokenizer()
  				__magic_name__	:	List[str]			     =  self.get_rust_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  				self.assertListEqual(
  				    [rust_tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  
  
  
  
  
  
  			@slow
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	int			     =  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)
  
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("sequence builders"		,     add_special_tokens=_a	)
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("multi-sequence build"		,     add_special_tokens=_a	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.build_inputs_with_special_tokens(_a	)
  				__magic_name__	:	Tuple			     =  tokenizer.build_inputs_with_special_tokens(_a		,     _a	)
  
  				assert encoded_sentence == [101] + text + [102]
  				assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	str			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Optional[Any]			     =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  						__magic_name__	:	Any			     =  tokenizer_r.encode_plus(
  						    _a		,     return_attention_mask=_a		,     return_token_type_ids=_a		,     return_offsets_mapping=_a		,     add_special_tokens=_a		,     )
  
  						__magic_name__	:	Any			     =  tokenizer_r.do_lower_case if hasattr(_a		,     "do_lower_case"	) else False
  						__magic_name__	:	int			     =  (
  						    [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "A"),
  						        ((1, 2), ","),
  						        ((3, 5), "na"),
  						        ((5, 6), "##ï"),
  						        ((6, 8), "##ve"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "Allen"),
  						        ((21, 23), "##NL"),
  						        ((23, 24), "##P"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						    if not do_lower_case
  						    else [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "a"),
  						        ((1, 2), ","),
  						        ((3, 8), "naive"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "allen"),
  						        ((21, 23), "##nl"),
  						        ((23, 24), "##p"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						)
  
  						self.assertEqual(
  						    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
  						self.assertEqual([e[0] for e in expected_results]		,     tokens["offset_mapping"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  ["的", "人", "有"]
  				__magic_name__	:	List[str]			     =  "".join(_a	)
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	Optional[int]			     =  True
  						__magic_name__	:	Optional[int]			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	Union[str, Any]			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Tuple			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that each Chinese character is not preceded by "##"
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  
  						__magic_name__	:	Tuple			     =  False
  						__magic_name__	:	Tuple			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	str			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Dict			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	Dict			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that only the first Chinese character is not preceded by "##".
  						__magic_name__	:	Tuple			     =  [
  						    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a	)
  						]
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  