code: 
  
  
  
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  @require_tokenizers
  class 		_snake_case      (			_a					,unittest.TestCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			ElectraTokenizer
   lowerCamelCase_          =			ElectraTokenizerFast
   lowerCamelCase_          =			True
   lowerCamelCase_          =			True
   lowerCamelCase_          =			filter_non_english
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     super().setUp()
  
     _A           =    [
         "[UNK]",
         "[CLS]",
         "[SEP]",
         "[PAD]",
         "[MASK]",
         "want",
         "##want",
         "##ed",
         "wa",
         "un",
         "runn",
         "##ing",
         ",",
         "low",
         "lowest",
     ]
     _A           =    os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''vocab_file''']					)
     with open(self.vocab_file	,    '''w'''	,    encoding='''utf-8'''					) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
   def lowercase_		(						self	,    a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    "UNwant\u00E9d,running"
     _A           =    "unwanted, running"
     return input_text, output_text
   def lowercase_		(						self					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.tokenizer_class(self.vocab_file					)
  
     _A           =    tokenizer.tokenize('''UNwant\u00E9d,running'''					)
     self.assertListEqual(lowerCamelCase__	,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__					)	,    [9, 6, 7, 1_2, 1_0, 1_1]					)
   def lowercase_		(						self					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     if not self.test_rust_tokenizer:
       return
  
     _A           =    self.get_tokenizer()
     _A           =    self.get_rust_tokenizer()
  
     _A           =    "UNwant\u00E9d,running"
  
     _A           =    tokenizer.tokenize(lowerCamelCase__					)
     _A           =    rust_tokenizer.tokenize(lowerCamelCase__					)
     self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
  
     _A           =    tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
     _A           =    rust_tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
     self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
  
     _A           =    self.get_rust_tokenizer()
     _A           =    tokenizer.encode(lowerCamelCase__					)
     _A           =    rust_tokenizer.encode(lowerCamelCase__					)
     self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
  
     # With lower casing
     _A           =    self.get_tokenizer(do_lower_case=lowerCamelCase__					)
     _A           =    self.get_rust_tokenizer(do_lower_case=lowerCamelCase__					)
  
     _A           =    "UNwant\u00E9d,running"
  
     _A           =    tokenizer.tokenize(lowerCamelCase__					)
     _A           =    rust_tokenizer.tokenize(lowerCamelCase__					)
     self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
  
     _A           =    tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
     _A           =    rust_tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
     self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
  
     _A           =    self.get_rust_tokenizer()
     _A           =    tokenizer.encode(lowerCamelCase__					)
     _A           =    rust_tokenizer.encode(lowerCamelCase__					)
     self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer()
  
     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)	,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
   def lowercase_		(						self					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''h\u00E9llo''']					)
   def lowercase_		(						self					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    never_split=['''[UNK]''']					)
  
     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
     _A           =    {}
     for i, token in enumerate(lowerCamelCase__					):
       _A           =    i
     _A           =    WordpieceTokenizer(vocab=lowerCamelCase__	,    unk_token='''[UNK]'''					)
  
     self.assertListEqual(tokenizer.tokenize(''''''					)	,    []					)
  
     self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)	,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)
  
     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)	,    ['''[UNK]''', '''runn''', '''##ing''']					)
   def lowercase_		(						self					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     self.assertTrue(_is_whitespace(''' '''					)					)
     self.assertTrue(_is_whitespace('''\t'''					)					)
     self.assertTrue(_is_whitespace('''\r'''					)					)
     self.assertTrue(_is_whitespace('''\n'''					)					)
     self.assertTrue(_is_whitespace('''\u00A0'''					)					)
  
     self.assertFalse(_is_whitespace('''A'''					)					)
     self.assertFalse(_is_whitespace('''-'''					)					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     self.assertTrue(_is_control('''\u0005'''					)					)
  
     self.assertFalse(_is_control('''A'''					)					)
     self.assertFalse(_is_control(''' '''					)					)
     self.assertFalse(_is_control('''\t'''					)					)
     self.assertFalse(_is_control('''\r'''					)					)
   def lowercase_		(						self					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     self.assertTrue(_is_punctuation('''-'''					)					)
     self.assertTrue(_is_punctuation('''$'''					)					)
     self.assertTrue(_is_punctuation('''`'''					)					)
     self.assertTrue(_is_punctuation('''.'''					)					)
  
     self.assertFalse(_is_punctuation('''A'''					)					)
     self.assertFalse(_is_punctuation(''' '''					)					)
   def lowercase_		(						self					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.get_tokenizer()
     _A           =    self.get_rust_tokenizer()
  
     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(lowerCamelCase__					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)
  
     self.assertListEqual(
         [rust_tokenizer.tokenize(lowerCamelCase__					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)
   @slow
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''					)
  
     _A           =    tokenizer.encode('''sequence builders'''	,    add_special_tokens=lowerCamelCase__					)
     _A           =    tokenizer.encode('''multi-sequence build'''	,    add_special_tokens=lowerCamelCase__					)
  
     _A           =    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__					)
     _A           =    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__	,    lowerCamelCase__					)
  
     assert encoded_sentence == [1_0_1] + text + [1_0_2]
     assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
   def lowercase_		(						self					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
         _A           =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
  
         _A           =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
         _A           =    tokenizer_r.encode_plus(
             lowerCamelCase__	,    return_attention_mask=lowerCamelCase__	,    return_token_type_ids=lowerCamelCase__	,    return_offsets_mapping=lowerCamelCase__	,    add_special_tokens=lowerCamelCase__	,    )
  
         _A           =    tokenizer_r.do_lower_case if hasattr(lowerCamelCase__	,    '''do_lower_case'''					) else False
         _A           =    (
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), "A"),
                 ((1, 2), ","),
                 ((3, 5), "na"),
                 ((5, 6), "##ï"),
                 ((6, 8), "##ve"),
                 ((9, 1_5), tokenizer_r.mask_token),
                 ((1_6, 2_1), "Allen"),
                 ((2_1, 2_3), "##NL"),
                 ((2_3, 2_4), "##P"),
                 ((2_5, 3_3), "sentence"),
                 ((3_3, 3_4), "."),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), "a"),
                 ((1, 2), ","),
                 ((3, 8), "naive"),
                 ((9, 1_5), tokenizer_r.mask_token),
                 ((1_6, 2_1), "allen"),
                 ((2_1, 2_3), "##nl"),
                 ((2_3, 2_4), "##p"),
                 ((2_5, 3_3), "sentence"),
                 ((3_3, 3_4), "."),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )
  
         self.assertEqual(
             [e[1] for e in expected_results]	,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
         self.assertEqual([e[0] for e in expected_results]	,    tokens['''offset_mapping''']					)
  
  
  
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    ["的", "人", "有"]
     _A           =    "".join(lowerCamelCase__					)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
         _A           =    True
         _A           =    self.tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
         _A           =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
  
         _A           =    tokenizer_p.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
         _A           =    tokenizer_r.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
  
         _A           =    tokenizer_r.convert_ids_to_tokens(lowerCamelCase__					)
         _A           =    tokenizer_p.convert_ids_to_tokens(lowerCamelCase__					)
  
         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
         self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
  
         _A           =    False
         _A           =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
         _A           =    self.tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
  
         _A           =    tokenizer_r.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
         _A           =    tokenizer_p.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
  
         _A           =    tokenizer_r.convert_ids_to_tokens(lowerCamelCase__					)
         _A           =    tokenizer_p.convert_ids_to_tokens(lowerCamelCase__					)
  
         # it is expected that only the first Chinese character is not preceded by "##".
         _A           =    [
             f'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase__					)
         ]
         self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
         self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
style_context: 
  
  
  
  from sklearn.metrics import accuracy_score
  
  import datasets
  
  
  __lowerCamelCase  				=					"""
  Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with:
  Accuracy = (TP + TN) / (TP + TN + FP + FN)
   Where:
  TP: True positive
  TN: True negative
  FP: False positive
  FN: False negative
  """
  
  
  __lowerCamelCase  				=					"""
  Args:
      predictions (`list` of `int`): Predicted labels.
      references (`list` of `int`): Ground truth labels.
      normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True.
      sample_weight (`list` of `float`): Sample weights Defaults to None.
  
  Returns:
      accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy.
  
  Examples:
  
      Example 1-A simple example
          >>> accuracy_metric = datasets.load_metric(\"accuracy\")
          >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])
          >>> print(results)
          {'accuracy': 0.5}
  
      Example 2-The same as Example 1, except with `normalize` set to `False`.
          >>> accuracy_metric = datasets.load_metric(\"accuracy\")
          >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False)
          >>> print(results)
          {'accuracy': 3.0}
  
      Example 3-The same as Example 1, except with `sample_weight` set.
          >>> accuracy_metric = datasets.load_metric(\"accuracy\")
          >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])
          >>> print(results)
          {'accuracy': 0.8778625954198473}
  """
  
  
  __lowerCamelCase  				=					"""
  @article{scikit-learn,
    title={Scikit-learn: Machine Learning in {P}ython},
    author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
           and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
           and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
           Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
    journal={Journal of Machine Learning Research},
    volume={12},
    pages={2825--2830},
    year={2011}
  }
  """
  
  
  
  @datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,_KWARGS_DESCRIPTION				)
  class 		_snake_case      (			datasets.Metric				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return datasets.MetricInfo(
         description=_DESCRIPTION	,    citation=_CITATION	,    inputs_description=_KWARGS_DESCRIPTION	,    features=datasets.Features(
             {
                 '''predictions''': datasets.Sequence(datasets.Value('''int32'''					)					),
                 '''references''': datasets.Sequence(datasets.Value('''int32'''					)					),
             }
             if self.config_name == '''multilabel'''
             else {
                 '''predictions''': datasets.Value('''int32'''					),
                 '''references''': datasets.Value('''int32'''					),
             }					)	,    reference_urls=['''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html''']	,    )
  
  
  
   def lowercase_		(						self	,    a	,    a	,    a=True	,    a=None					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return {
         "accuracy": float(
             accuracy_score(a	,    a	,    normalize=a	,    sample_weight=a					)					)
     }