



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 		_snake_case      (			_a					,unittest.TestCase				):





 """simple docstring"""



 lowerCamelCase_          =			ElectraTokenizer
 lowerCamelCase_          =			ElectraTokenizerFast
 lowerCamelCase_          =			True
 lowerCamelCase_          =			True
 lowerCamelCase_          =			filter_non_english
 def lowercase_		(						self					)	->							Dict:




   """simple docstring"""







   super().setUp()

   _A           =    [
       "[UNK]",
       "[CLS]",
       "[SEP]",
       "[PAD]",
       "[MASK]",
       "want",
       "##want",
       "##ed",
       "wa",
       "un",
       "runn",
       "##ing",
       ",",
       "low",
       "lowest",
   ]
   _A           =    os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''vocab_file''']					)
   with open(self.vocab_file	,    '''w'''	,    encoding='''utf-8'''					) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
 def lowercase_		(						self	,    a					)	->							int:




   """simple docstring"""







   _A           =    "UNwant\u00E9d,running"
   _A           =    "unwanted, running"
   return input_text, output_text
 def lowercase_		(						self					)	->							Union[str, Any]:




   """simple docstring"""







   _A           =    self.tokenizer_class(self.vocab_file					)

   _A           =    tokenizer.tokenize('''UNwant\u00E9d,running'''					)
   self.assertListEqual(lowerCamelCase__	,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
   self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__					)	,    [9, 6, 7, 1_2, 1_0, 1_1]					)
 def lowercase_		(						self					)	->							List[str]:




   """simple docstring"""







   if not self.test_rust_tokenizer:
     return

   _A           =    self.get_tokenizer()
   _A           =    self.get_rust_tokenizer()

   _A           =    "UNwant\u00E9d,running"

   _A           =    tokenizer.tokenize(lowerCamelCase__					)
   _A           =    rust_tokenizer.tokenize(lowerCamelCase__					)
   self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)

   _A           =    tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
   _A           =    rust_tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
   self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)

   _A           =    self.get_rust_tokenizer()
   _A           =    tokenizer.encode(lowerCamelCase__					)
   _A           =    rust_tokenizer.encode(lowerCamelCase__					)
   self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)

   # With lower casing
   _A           =    self.get_tokenizer(do_lower_case=lowerCamelCase__					)
   _A           =    self.get_rust_tokenizer(do_lower_case=lowerCamelCase__					)

   _A           =    "UNwant\u00E9d,running"

   _A           =    tokenizer.tokenize(lowerCamelCase__					)
   _A           =    rust_tokenizer.tokenize(lowerCamelCase__					)
   self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)

   _A           =    tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
   _A           =    rust_tokenizer.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
   self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)

   _A           =    self.get_rust_tokenizer()
   _A           =    tokenizer.encode(lowerCamelCase__					)
   _A           =    rust_tokenizer.encode(lowerCamelCase__					)
   self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
 def lowercase_		(						self					)	->							Any:




   """simple docstring"""







   _A           =    BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)	,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
 def lowercase_		(						self					)	->							Dict:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
 def lowercase_		(						self					)	->							List[str]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''h\u00E9llo''']					)
 def lowercase_		(						self					)	->							List[str]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
 def lowercase_		(						self					)	->							Any:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)	,    ['''hello''']					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    strip_accents=lowerCamelCase__					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)	,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   _A           =    BasicTokenizer(do_lower_case=lowerCamelCase__	,    never_split=['''[UNK]''']					)

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)	,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   _A           =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

   _A           =    {}
   for i, token in enumerate(lowerCamelCase__					):
     _A           =    i
   _A           =    WordpieceTokenizer(vocab=lowerCamelCase__	,    unk_token='''[UNK]'''					)

   self.assertListEqual(tokenizer.tokenize(''''''					)	,    []					)

   self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)	,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

   self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)	,    ['''[UNK]''', '''runn''', '''##ing''']					)
 def lowercase_		(						self					)	->							Union[str, Any]:




   """simple docstring"""







   self.assertTrue(_is_whitespace(''' '''					)					)
   self.assertTrue(_is_whitespace('''\t'''					)					)
   self.assertTrue(_is_whitespace('''\r'''					)					)
   self.assertTrue(_is_whitespace('''\n'''					)					)
   self.assertTrue(_is_whitespace('''\u00A0'''					)					)

   self.assertFalse(_is_whitespace('''A'''					)					)
   self.assertFalse(_is_whitespace('''-'''					)					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   self.assertTrue(_is_control('''\u0005'''					)					)

   self.assertFalse(_is_control('''A'''					)					)
   self.assertFalse(_is_control(''' '''					)					)
   self.assertFalse(_is_control('''\t'''					)					)
   self.assertFalse(_is_control('''\r'''					)					)
 def lowercase_		(						self					)	->							Tuple:




   """simple docstring"""







   self.assertTrue(_is_punctuation('''-'''					)					)
   self.assertTrue(_is_punctuation('''$'''					)					)
   self.assertTrue(_is_punctuation('''`'''					)					)
   self.assertTrue(_is_punctuation('''.'''					)					)

   self.assertFalse(_is_punctuation('''A'''					)					)
   self.assertFalse(_is_punctuation(''' '''					)					)
 def lowercase_		(						self					)	->							List[Any]:




   """simple docstring"""







   _A           =    self.get_tokenizer()
   _A           =    self.get_rust_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(lowerCamelCase__					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)

   self.assertListEqual(
       [rust_tokenizer.tokenize(lowerCamelCase__					) for t in ['''Test''', '''\xad''', '''test''']]	,    [['''[UNK]'''], [], ['''[UNK]''']]					)
 @slow
 def lowercase_		(						self					)	->							Any:




   """simple docstring"""







   _A           =    self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''					)

   _A           =    tokenizer.encode('''sequence builders'''	,    add_special_tokens=lowerCamelCase__					)
   _A           =    tokenizer.encode('''multi-sequence build'''	,    add_special_tokens=lowerCamelCase__					)

   _A           =    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__					)
   _A           =    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__	,    lowerCamelCase__					)

   assert encoded_sentence == [1_0_1] + text + [1_0_2]
   assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
 def lowercase_		(						self					)	->							List[str]:




   """simple docstring"""







   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
       _A           =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)

       _A           =    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
       _A           =    tokenizer_r.encode_plus(
           lowerCamelCase__	,    return_attention_mask=lowerCamelCase__	,    return_token_type_ids=lowerCamelCase__	,    return_offsets_mapping=lowerCamelCase__	,    add_special_tokens=lowerCamelCase__	,    )

       _A           =    tokenizer_r.do_lower_case if hasattr(lowerCamelCase__	,    '''do_lower_case'''					) else False
       _A           =    (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "A"),
               ((1, 2), ","),
               ((3, 5), "na"),
               ((5, 6), "##ï"),
               ((6, 8), "##ve"),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), "Allen"),
               ((2_1, 2_3), "##NL"),
               ((2_3, 2_4), "##P"),
               ((2_5, 3_3), "sentence"),
               ((3_3, 3_4), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "a"),
               ((1, 2), ","),
               ((3, 8), "naive"),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), "allen"),
               ((2_1, 2_3), "##nl"),
               ((2_3, 2_4), "##p"),
               ((2_5, 3_3), "sentence"),
               ((3_3, 3_4), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]	,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
       self.assertEqual([e[0] for e in expected_results]	,    tokens['''offset_mapping''']					)



 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   _A           =    ["的", "人", "有"]
   _A           =    "".join(lowerCamelCase__					)
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
       _A           =    True
       _A           =    self.tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
       _A           =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)

       _A           =    tokenizer_p.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
       _A           =    tokenizer_r.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)

       _A           =    tokenizer_r.convert_ids_to_tokens(lowerCamelCase__					)
       _A           =    tokenizer_p.convert_ids_to_tokens(lowerCamelCase__					)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
       self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)

       _A           =    False
       _A           =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)
       _A           =    self.tokenizer_class.from_pretrained(lowerCamelCase__	,    **lowerCamelCase__					)

       _A           =    tokenizer_r.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)
       _A           =    tokenizer_p.encode(lowerCamelCase__	,    add_special_tokens=lowerCamelCase__					)

       _A           =    tokenizer_r.convert_ids_to_tokens(lowerCamelCase__					)
       _A           =    tokenizer_p.convert_ids_to_tokens(lowerCamelCase__					)

       # it is expected that only the first Chinese character is not preceded by "##".
       _A           =    [
           f'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase__					)
       ]
       self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)
       self.assertListEqual(lowerCamelCase__	,    lowerCamelCase__					)




from sklearn.metrics import accuracy_score

import datasets


__lowerCamelCase  				=					"""
Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with:
Accuracy = (TP + TN) / (TP + TN + FP + FN)
 Where:
TP: True positive
TN: True negative
FP: False positive
FN: False negative
"""


__lowerCamelCase  				=					"""
Args:
    predictions (`list` of `int`): Predicted labels.
    references (`list` of `int`): Ground truth labels.
    normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True.
    sample_weight (`list` of `float`): Sample weights Defaults to None.

Returns:
    accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy.

Examples:

    Example 1-A simple example
        >>> accuracy_metric = datasets.load_metric(\"accuracy\")
        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])
        >>> print(results)
        {'accuracy': 0.5}

    Example 2-The same as Example 1, except with `normalize` set to `False`.
        >>> accuracy_metric = datasets.load_metric(\"accuracy\")
        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False)
        >>> print(results)
        {'accuracy': 3.0}

    Example 3-The same as Example 1, except with `sample_weight` set.
        >>> accuracy_metric = datasets.load_metric(\"accuracy\")
        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])
        >>> print(results)
        {'accuracy': 0.8778625954198473}
"""


__lowerCamelCase  				=					"""
@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}
"""



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,_KWARGS_DESCRIPTION				)
class 		_snake_case      (			datasets.Metric				):





 """simple docstring"""



 def lowercase_		(						self					)	->							Optional[Any]:




   """simple docstring"""







   return datasets.MetricInfo(
       description=_DESCRIPTION	,    citation=_CITATION	,    inputs_description=_KWARGS_DESCRIPTION	,    features=datasets.Features(
           {
               '''predictions''': datasets.Sequence(datasets.Value('''int32'''					)					),
               '''references''': datasets.Sequence(datasets.Value('''int32'''					)					),
           }
           if self.config_name == '''multilabel'''
           else {
               '''predictions''': datasets.Value('''int32'''					),
               '''references''': datasets.Value('''int32'''					),
           }					)	,    reference_urls=['''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html''']	,    )



 def lowercase_		(						self	,    a	,    a	,    a=True	,    a=None					)	->							Dict:




   """simple docstring"""







   return {
       "accuracy": float(
           accuracy_score(a	,    a	,    normalize=a	,    sample_weight=a					)					)
   }