code: 
  
  
  
  
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class 							_lowerCAmelCase		(     UpperCamelCase__  ,   unittest.TestCase	):
  				lowerCamelCase__									=    ElectraTokenizer
  				lowerCamelCase__									=    ElectraTokenizerFast
  				lowerCamelCase__									=    True
  				lowerCamelCase__									=    True
  				lowerCamelCase__									=    filter_non_english
  
  
  
  
  				def     __a		( self							)    ->      Tuple:
  											super().setUp()
  
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =[
  											    '''[UNK]''',
  											    '''[CLS]''',
  											    '''[SEP]''',
  											    '''[PAD]''',
  											    '''[MASK]''',
  											    '''want''',
  											    '''##want''',
  											    '''##ed''',
  											    '''wa''',
  											    '''un''',
  											    '''runn''',
  											    '''##ing''',
  											    ''',''',
  											    '''low''',
  											    '''lowest''',
  											]
  											SCREAMING_SNAKE_CASE			: str	     =os.path.join(self.tmpdirname		,		VOCAB_FILES_NAMES['''vocab_file''']							)
  											with open(self.vocab_file		,		'''w'''		,		encoding='''utf-8'''							) as vocab_writer:
  																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
  
  
  
  
  				def     __a		( self		,		snake_case_							)    ->      Tuple:
  											SCREAMING_SNAKE_CASE			: int	     ='''UNwant\u00E9d,running'''
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     ='''unwanted, running'''
  											return input_text, output_text
  
  
  
  
  				def     __a		( self							)    ->      Tuple:
  											SCREAMING_SNAKE_CASE			: Tuple	     =self.tokenizer_class(self.vocab_file							)
  
  											SCREAMING_SNAKE_CASE			: List[Any]	     =tokenizer.tokenize('''UNwant\u00E9d,running'''							)
  											self.assertListEqual(snake_case_		,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
  											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_							)		,		[9, 6, 7, 12, 10, 11]							)
  
  
  
  
  				def     __a		( self							)    ->      Tuple:
  											if not self.test_rust_tokenizer:
  																		return
  
  											SCREAMING_SNAKE_CASE			: Any	     =self.get_tokenizer()
  											SCREAMING_SNAKE_CASE			: Any	     =self.get_rust_tokenizer()
  
  											SCREAMING_SNAKE_CASE			: List[Any]	     ='''UNwant\u00E9d,running'''
  
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =tokenizer.tokenize(snake_case_							)
  											SCREAMING_SNAKE_CASE			: str	     =rust_tokenizer.tokenize(snake_case_							)
  											self.assertListEqual(snake_case_		,		snake_case_							)
  
  											SCREAMING_SNAKE_CASE			: List[Any]	     =tokenizer.encode(snake_case_		,		add_special_tokens=snake_case_							)
  											SCREAMING_SNAKE_CASE			: List[Any]	     =rust_tokenizer.encode(snake_case_		,		add_special_tokens=snake_case_							)
  											self.assertListEqual(snake_case_		,		snake_case_							)
  
  											SCREAMING_SNAKE_CASE			: int	     =self.get_rust_tokenizer()
  											SCREAMING_SNAKE_CASE			: Dict	     =tokenizer.encode(snake_case_							)
  											SCREAMING_SNAKE_CASE			: Dict	     =rust_tokenizer.encode(snake_case_							)
  											self.assertListEqual(snake_case_		,		snake_case_							)
  
  											# With lower casing
  											SCREAMING_SNAKE_CASE			: List[str]	     =self.get_tokenizer(do_lower_case=snake_case_							)
  											SCREAMING_SNAKE_CASE			: int	     =self.get_rust_tokenizer(do_lower_case=snake_case_							)
  
  											SCREAMING_SNAKE_CASE			: Optional[int]	     ='''UNwant\u00E9d,running'''
  
  											SCREAMING_SNAKE_CASE			: int	     =tokenizer.tokenize(snake_case_							)
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =rust_tokenizer.tokenize(snake_case_							)
  											self.assertListEqual(snake_case_		,		snake_case_							)
  
  											SCREAMING_SNAKE_CASE			: List[Any]	     =tokenizer.encode(snake_case_		,		add_special_tokens=snake_case_							)
  											SCREAMING_SNAKE_CASE			: Any	     =rust_tokenizer.encode(snake_case_		,		add_special_tokens=snake_case_							)
  											self.assertListEqual(snake_case_		,		snake_case_							)
  
  											SCREAMING_SNAKE_CASE			: str	     =self.get_rust_tokenizer()
  											SCREAMING_SNAKE_CASE			: Tuple	     =tokenizer.encode(snake_case_							)
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =rust_tokenizer.encode(snake_case_							)
  											self.assertListEqual(snake_case_		,		snake_case_							)
  
  
  
  
  				def     __a		( self							)    ->      Dict:
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =BasicTokenizer()
  
  											self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)		,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)
  
  
  
  
  				def     __a		( self							)    ->      Tuple:
  											SCREAMING_SNAKE_CASE			: int	     =BasicTokenizer(do_lower_case=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)		,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,		['''hello''']							)
  
  
  
  
  				def     __a		( self							)    ->      List[str]:
  											SCREAMING_SNAKE_CASE			: str	     =BasicTokenizer(do_lower_case=snake_case_		,		strip_accents=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,		['''h\u00E9llo''']							)
  
  
  
  
  				def     __a		( self							)    ->      List[Any]:
  											SCREAMING_SNAKE_CASE			: List[Any]	     =BasicTokenizer(do_lower_case=snake_case_		,		strip_accents=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,		['''hello''']							)
  
  
  
  
  				def     __a		( self							)    ->      Any:
  											SCREAMING_SNAKE_CASE			: int	     =BasicTokenizer(do_lower_case=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
  											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,		['''hello''']							)
  
  
  
  
  				def     __a		( self							)    ->      Tuple:
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =BasicTokenizer(do_lower_case=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)		,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)
  
  
  
  
  				def     __a		( self							)    ->      Union[str, Any]:
  											SCREAMING_SNAKE_CASE			: Tuple	     =BasicTokenizer(do_lower_case=snake_case_		,		strip_accents=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)
  
  
  
  
  				def     __a		( self							)    ->      Any:
  											SCREAMING_SNAKE_CASE			: int	     =BasicTokenizer(do_lower_case=snake_case_		,		strip_accents=snake_case_							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)
  
  
  
  
  				def     __a		( self							)    ->      List[str]:
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =BasicTokenizer(do_lower_case=snake_case_		,		never_split=['''[UNK]''']							)
  
  											self.assertListEqual(
  											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)		,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)
  
  
  
  
  				def     __a		( self							)    ->      str:
  											SCREAMING_SNAKE_CASE			: List[Any]	     =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
  											SCREAMING_SNAKE_CASE			: str	     ={}
  											for i, token in enumerate(snake_case_							):
  																		SCREAMING_SNAKE_CASE			: List[str]	     =i
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =WordpieceTokenizer(vocab=snake_case_		,		unk_token='''[UNK]'''							)
  
  											self.assertListEqual(tokenizer.tokenize(''''''							)		,		[]							)
  
  											self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)		,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)
  
  											self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)		,		['''[UNK]''', '''runn''', '''##ing''']							)
  
  
  
  
  				def     __a		( self							)    ->      str:
  											self.assertTrue(_is_whitespace(''' '''							)							)
  											self.assertTrue(_is_whitespace('''\t'''							)							)
  											self.assertTrue(_is_whitespace('''\r'''							)							)
  											self.assertTrue(_is_whitespace('''\n'''							)							)
  											self.assertTrue(_is_whitespace('''\u00A0'''							)							)
  
  											self.assertFalse(_is_whitespace('''A'''							)							)
  											self.assertFalse(_is_whitespace('''-'''							)							)
  
  
  
  
  				def     __a		( self							)    ->      List[Any]:
  											self.assertTrue(_is_control('''\u0005'''							)							)
  
  											self.assertFalse(_is_control('''A'''							)							)
  											self.assertFalse(_is_control(''' '''							)							)
  											self.assertFalse(_is_control('''\t'''							)							)
  											self.assertFalse(_is_control('''\r'''							)							)
  
  
  
  
  				def     __a		( self							)    ->      Dict:
  											self.assertTrue(_is_punctuation('''-'''							)							)
  											self.assertTrue(_is_punctuation('''$'''							)							)
  											self.assertTrue(_is_punctuation('''`'''							)							)
  											self.assertTrue(_is_punctuation('''.'''							)							)
  
  											self.assertFalse(_is_punctuation('''A'''							)							)
  											self.assertFalse(_is_punctuation(''' '''							)							)
  
  
  
  
  				def     __a		( self							)    ->      str:
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =self.get_tokenizer()
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =self.get_rust_tokenizer()
  
  											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  											self.assertListEqual([tokenizer.tokenize(snake_case_							) for t in ['''Test''', '''\xad''', '''test''']]		,		[['''[UNK]'''], [], ['''[UNK]''']]							)
  
  											self.assertListEqual(
  											    [rust_tokenizer.tokenize(snake_case_							) for t in ['''Test''', '''\xad''', '''test''']]		,		[['''[UNK]'''], [], ['''[UNK]''']]							)
  
  
  
  
  				@slow
  				def     __a		( self							)    ->      Union[str, Any]:
  											SCREAMING_SNAKE_CASE			: Any	     =self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''							)
  
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =tokenizer.encode('''sequence builders'''		,		add_special_tokens=snake_case_							)
  											SCREAMING_SNAKE_CASE			: str	     =tokenizer.encode('''multi-sequence build'''		,		add_special_tokens=snake_case_							)
  
  											SCREAMING_SNAKE_CASE			: Dict	     =tokenizer.build_inputs_with_special_tokens(snake_case_							)
  											SCREAMING_SNAKE_CASE			: Any	     =tokenizer.build_inputs_with_special_tokens(snake_case_		,		snake_case_							)
  
  											assert encoded_sentence == [101] + text + [102]
  											assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  				def     __a		( self							)    ->      Union[str, Any]:
  											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  																		with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
  																									SCREAMING_SNAKE_CASE			: List[Any]	     =self.rust_tokenizer_class.from_pretrained(snake_case_		,		**snake_case_							)
  
  																									SCREAMING_SNAKE_CASE			: Dict	     =F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
  																									SCREAMING_SNAKE_CASE			: List[str]	     =tokenizer_r.encode_plus(
  																									    snake_case_		,		return_attention_mask=snake_case_		,		return_token_type_ids=snake_case_		,		return_offsets_mapping=snake_case_		,		add_special_tokens=snake_case_		,		)
  
  																									SCREAMING_SNAKE_CASE			: List[str]	     =tokenizer_r.do_lower_case if hasattr(snake_case_		,		'''do_lower_case'''							) else False
  																									SCREAMING_SNAKE_CASE			: List[str]	     =(
  																									    [
  																									        ((0, 0), tokenizer_r.cls_token),
  																									        ((0, 1), '''A'''),
  																									        ((1, 2), ''','''),
  																									        ((3, 5), '''na'''),
  																									        ((5, 6), '''##ï'''),
  																									        ((6, 8), '''##ve'''),
  																									        ((9, 15), tokenizer_r.mask_token),
  																									        ((16, 21), '''Allen'''),
  																									        ((21, 23), '''##NL'''),
  																									        ((23, 24), '''##P'''),
  																									        ((25, 33), '''sentence'''),
  																									        ((33, 34), '''.'''),
  																									        ((0, 0), tokenizer_r.sep_token),
  																									    ]
  																									    if not do_lower_case
  																									    else [
  																									        ((0, 0), tokenizer_r.cls_token),
  																									        ((0, 1), '''a'''),
  																									        ((1, 2), ''','''),
  																									        ((3, 8), '''naive'''),
  																									        ((9, 15), tokenizer_r.mask_token),
  																									        ((16, 21), '''allen'''),
  																									        ((21, 23), '''##nl'''),
  																									        ((23, 24), '''##p'''),
  																									        ((25, 33), '''sentence'''),
  																									        ((33, 34), '''.'''),
  																									        ((0, 0), tokenizer_r.sep_token),
  																									    ]
  																									)
  
  																									self.assertEqual(
  																									    [e[1] for e in expected_results]		,		tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
  																									self.assertEqual([e[0] for e in expected_results]		,		tokens['''offset_mapping''']							)
  
  
  
  
  
  
  
  				def     __a		( self							)    ->      Optional[Any]:
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =['''的''', '''人''', '''有''']
  											SCREAMING_SNAKE_CASE			: List[str]	     =''''''.join(snake_case_							)
  											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  																		with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
  																									SCREAMING_SNAKE_CASE			: str	     =True
  																									SCREAMING_SNAKE_CASE			: Optional[Any]	     =self.tokenizer_class.from_pretrained(snake_case_		,		**snake_case_							)
  																									SCREAMING_SNAKE_CASE			: Optional[int]	     =self.rust_tokenizer_class.from_pretrained(snake_case_		,		**snake_case_							)
  
  																									SCREAMING_SNAKE_CASE			: int	     =tokenizer_p.encode(snake_case_		,		add_special_tokens=snake_case_							)
  																									SCREAMING_SNAKE_CASE			: int	     =tokenizer_r.encode(snake_case_		,		add_special_tokens=snake_case_							)
  
  																									SCREAMING_SNAKE_CASE			: Union[str, Any]	     =tokenizer_r.convert_ids_to_tokens(snake_case_							)
  																									SCREAMING_SNAKE_CASE			: Dict	     =tokenizer_p.convert_ids_to_tokens(snake_case_							)
  
  																									# it is expected that each Chinese character is not preceded by "##"
  																									self.assertListEqual(snake_case_		,		snake_case_							)
  																									self.assertListEqual(snake_case_		,		snake_case_							)
  
  																									SCREAMING_SNAKE_CASE			: Any	     =False
  																									SCREAMING_SNAKE_CASE			: Optional[int]	     =self.rust_tokenizer_class.from_pretrained(snake_case_		,		**snake_case_							)
  																									SCREAMING_SNAKE_CASE			: Dict	     =self.tokenizer_class.from_pretrained(snake_case_		,		**snake_case_							)
  
  																									SCREAMING_SNAKE_CASE			: str	     =tokenizer_r.encode(snake_case_		,		add_special_tokens=snake_case_							)
  																									SCREAMING_SNAKE_CASE			: Optional[int]	     =tokenizer_p.encode(snake_case_		,		add_special_tokens=snake_case_							)
  
  																									SCREAMING_SNAKE_CASE			: Optional[int]	     =tokenizer_r.convert_ids_to_tokens(snake_case_							)
  																									SCREAMING_SNAKE_CASE			: Any	     =tokenizer_p.convert_ids_to_tokens(snake_case_							)
  
  																									# it is expected that only the first Chinese character is not preceded by "##".
  																									SCREAMING_SNAKE_CASE			: Any	     =[
  																									    F'##{token}' if idx != 0 else token for idx, token in enumerate(snake_case_							)
  																									]
  																									self.assertListEqual(snake_case_		,		snake_case_							)
  																									self.assertListEqual(snake_case_		,		snake_case_							)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_torch_available,
  )
  
  
  _A							    =	{"""configuration_regnet""": ["""REGNET_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RegNetConfig"""]}
  
  try:
  							if not is_torch_available():
  														raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  							pass
  else:
  							_A							    =	[
  							    """REGNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
  							    """RegNetForImageClassification""",
  							    """RegNetModel""",
  							    """RegNetPreTrainedModel""",
  							]
  
  try:
  							if not is_tf_available():
  														raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  							pass
  else:
  							_A							    =	[
  							    """TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
  							    """TFRegNetForImageClassification""",
  							    """TFRegNetModel""",
  							    """TFRegNetPreTrainedModel""",
  							]
  
  try:
  							if not is_flax_available():
  														raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  							pass
  else:
  							_A							    =	[
  							    """FlaxRegNetForImageClassification""",
  							    """FlaxRegNetModel""",
  							    """FlaxRegNetPreTrainedModel""",
  							]
  
  
  if TYPE_CHECKING:
  							from .configuration_regnet import REGNET_PRETRAINED_CONFIG_ARCHIVE_MAP, RegNetConfig
  
  							try:
  														if not is_torch_available():
  																					raise OptionalDependencyNotAvailable()
  							except OptionalDependencyNotAvailable:
  														pass
  							else:
  														from .modeling_regnet import (
  														    REGNET_PRETRAINED_MODEL_ARCHIVE_LIST,
  														    RegNetForImageClassification,
  														    RegNetModel,
  														    RegNetPreTrainedModel,
  														)
  
  							try:
  														if not is_tf_available():
  																					raise OptionalDependencyNotAvailable()
  							except OptionalDependencyNotAvailable:
  														pass
  							else:
  														from .modeling_tf_regnet import (
  														    TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST,
  														    TFRegNetForImageClassification,
  														    TFRegNetModel,
  														    TFRegNetPreTrainedModel,
  														)
  
  							try:
  														if not is_flax_available():
  																					raise OptionalDependencyNotAvailable()
  							except OptionalDependencyNotAvailable:
  														pass
  							else:
  														from .modeling_flax_regnet import (
  														    FlaxRegNetForImageClassification,
  														    FlaxRegNetModel,
  														    FlaxRegNetPreTrainedModel,
  														)
  
  
  else:
  							import sys
  
  							_A							    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure)
  
  
  
  
  
  
  