





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 			A_ (a_  ,							unittest.TestCase    ):
	UpperCAmelCase__      =    ElectraTokenizer
	UpperCAmelCase__      =    ElectraTokenizerFast
	UpperCAmelCase__      =    True
	UpperCAmelCase__      =    True
	UpperCAmelCase__      =    filter_non_english
	def   _lowercase       (					self	):






				'''simple docstring'''







				super().setUp()

				UpperCAmelCase     	=       [
				    '''[UNK]''',
				    '''[CLS]''',
				    '''[SEP]''',
				    '''[PAD]''',
				    '''[MASK]''',
				    '''want''',
				    '''##want''',
				    '''##ed''',
				    '''wa''',
				    '''un''',
				    '''runn''',
				    '''##ing''',
				    ''',''',
				    '''low''',
				    '''lowest''',
				]
				UpperCAmelCase     	=       os.path.join(self.tmpdirname							,      VOCAB_FILES_NAMES['''vocab_file''']	)
				with open(self.vocab_file							,      '''w'''							,      encoding='''utf-8'''	) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
	def   _lowercase       (					self							,      _A	):






				'''simple docstring'''







				UpperCAmelCase     	=       '''UNwant\u00E9d,running'''
				UpperCAmelCase     	=       '''unwanted, running'''
				return input_text, output_text
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       self.tokenizer_class(self.vocab_file	)

				UpperCAmelCase     	=       tokenizer.tokenize('''UNwant\u00E9d,running'''	)
				self.assertListEqual(_A							,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)							,      [9, 6, 7, 1_2, 1_0, 1_1]	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase     	=       self.get_tokenizer()
				UpperCAmelCase     	=       self.get_rust_tokenizer()

				UpperCAmelCase     	=       '''UNwant\u00E9d,running'''

				UpperCAmelCase     	=       tokenizer.tokenize(_A	)
				UpperCAmelCase     	=       rust_tokenizer.tokenize(_A	)
				self.assertListEqual(_A							,      _A	)

				UpperCAmelCase     	=       tokenizer.encode(_A							,      add_special_tokens=_A	)
				UpperCAmelCase     	=       rust_tokenizer.encode(_A							,      add_special_tokens=_A	)
				self.assertListEqual(_A							,      _A	)

				UpperCAmelCase     	=       self.get_rust_tokenizer()
				UpperCAmelCase     	=       tokenizer.encode(_A	)
				UpperCAmelCase     	=       rust_tokenizer.encode(_A	)
				self.assertListEqual(_A							,      _A	)

				# With lower casing
				UpperCAmelCase     	=       self.get_tokenizer(do_lower_case=_A	)
				UpperCAmelCase     	=       self.get_rust_tokenizer(do_lower_case=_A	)

				UpperCAmelCase     	=       '''UNwant\u00E9d,running'''

				UpperCAmelCase     	=       tokenizer.tokenize(_A	)
				UpperCAmelCase     	=       rust_tokenizer.tokenize(_A	)
				self.assertListEqual(_A							,      _A	)

				UpperCAmelCase     	=       tokenizer.encode(_A							,      add_special_tokens=_A	)
				UpperCAmelCase     	=       rust_tokenizer.encode(_A							,      add_special_tokens=_A	)
				self.assertListEqual(_A							,      _A	)

				UpperCAmelCase     	=       self.get_rust_tokenizer()
				UpperCAmelCase     	=       tokenizer.encode(_A	)
				UpperCAmelCase     	=       rust_tokenizer.encode(_A	)
				self.assertListEqual(_A							,      _A	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)							,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)							,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)							,      ['''hello''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A							,      strip_accents=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)							,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)							,      ['''h\u00E9llo''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A							,      strip_accents=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)							,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)							,      ['''hello''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)							,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)							,      ['''hello''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)							,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A							,      strip_accents=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)							,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A							,      strip_accents=_A	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)							,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       BasicTokenizer(do_lower_case=_A							,      never_split=['''[UNK]''']	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)							,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

				UpperCAmelCase     	=       {}
				for i, token in enumerate(_A	):
							UpperCAmelCase     	=       i
				UpperCAmelCase     	=       WordpieceTokenizer(vocab=_A							,      unk_token='''[UNK]'''	)

				self.assertListEqual(tokenizer.tokenize(''''''	)							,      []	)

				self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)							,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

				self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)							,      ['''[UNK]''', '''runn''', '''##ing''']	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				self.assertTrue(_is_whitespace(''' '''	)	)
				self.assertTrue(_is_whitespace('''\t'''	)	)
				self.assertTrue(_is_whitespace('''\r'''	)	)
				self.assertTrue(_is_whitespace('''\n'''	)	)
				self.assertTrue(_is_whitespace('''\u00A0'''	)	)

				self.assertFalse(_is_whitespace('''A'''	)	)
				self.assertFalse(_is_whitespace('''-'''	)	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				self.assertTrue(_is_control('''\u0005'''	)	)

				self.assertFalse(_is_control('''A'''	)	)
				self.assertFalse(_is_control(''' '''	)	)
				self.assertFalse(_is_control('''\t'''	)	)
				self.assertFalse(_is_control('''\r'''	)	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				self.assertTrue(_is_punctuation('''-'''	)	)
				self.assertTrue(_is_punctuation('''$'''	)	)
				self.assertTrue(_is_punctuation('''`'''	)	)
				self.assertTrue(_is_punctuation('''.'''	)	)

				self.assertFalse(_is_punctuation('''A'''	)	)
				self.assertFalse(_is_punctuation(''' '''	)	)
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       self.get_tokenizer()
				UpperCAmelCase     	=       self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_A	) for t in ['''Test''', '''\xad''', '''test''']]							,      [['''[UNK]'''], [], ['''[UNK]''']]	)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(_A	) for t in ['''Test''', '''\xad''', '''test''']]							,      [['''[UNK]'''], [], ['''[UNK]''']]	)
	@slow
	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''	)

				UpperCAmelCase     	=       tokenizer.encode('''sequence builders'''							,      add_special_tokens=_A	)
				UpperCAmelCase     	=       tokenizer.encode('''multi-sequence build'''							,      add_special_tokens=_A	)

				UpperCAmelCase     	=       tokenizer.build_inputs_with_special_tokens(_A	)
				UpperCAmelCase     	=       tokenizer.build_inputs_with_special_tokens(_A							,      _A	)

				assert encoded_sentence == [1_0_1] + text + [1_0_2]
				assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
	def   _lowercase       (					self	):






				'''simple docstring'''







				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
										UpperCAmelCase     	=       self.rust_tokenizer_class.from_pretrained(_A							,      **_A	)

										UpperCAmelCase     	=       F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase     	=       tokenizer_r.encode_plus(
										    _A							,      return_attention_mask=_A							,      return_token_type_ids=_A							,      return_offsets_mapping=_A							,      add_special_tokens=_A							,      )

										UpperCAmelCase     	=       tokenizer_r.do_lower_case if hasattr(_A							,      '''do_lower_case'''	) else False
										UpperCAmelCase     	=       (
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), '''A'''),
										        ((1, 2), ''','''),
										        ((3, 5), '''na'''),
										        ((5, 6), '''##ï'''),
										        ((6, 8), '''##ve'''),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), '''Allen'''),
										        ((2_1, 2_3), '''##NL'''),
										        ((2_3, 2_4), '''##P'''),
										        ((2_5, 3_3), '''sentence'''),
										        ((3_3, 3_4), '''.'''),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), '''a'''),
										        ((1, 2), ''','''),
										        ((3, 8), '''naive'''),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), '''allen'''),
										        ((2_1, 2_3), '''##nl'''),
										        ((2_3, 2_4), '''##p'''),
										        ((2_5, 3_3), '''sentence'''),
										        ((3_3, 3_4), '''.'''),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]							,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
										self.assertEqual([e[0] for e in expected_results]							,      tokens['''offset_mapping''']	)







	def   _lowercase       (					self	):






				'''simple docstring'''







				UpperCAmelCase     	=       ['''的''', '''人''', '''有''']
				UpperCAmelCase     	=       ''''''.join(_A	)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
										UpperCAmelCase     	=       True
										UpperCAmelCase     	=       self.tokenizer_class.from_pretrained(_A							,      **_A	)
										UpperCAmelCase     	=       self.rust_tokenizer_class.from_pretrained(_A							,      **_A	)

										UpperCAmelCase     	=       tokenizer_p.encode(_A							,      add_special_tokens=_A	)
										UpperCAmelCase     	=       tokenizer_r.encode(_A							,      add_special_tokens=_A	)

										UpperCAmelCase     	=       tokenizer_r.convert_ids_to_tokens(_A	)
										UpperCAmelCase     	=       tokenizer_p.convert_ids_to_tokens(_A	)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(_A							,      _A	)
										self.assertListEqual(_A							,      _A	)

										UpperCAmelCase     	=       False
										UpperCAmelCase     	=       self.rust_tokenizer_class.from_pretrained(_A							,      **_A	)
										UpperCAmelCase     	=       self.tokenizer_class.from_pretrained(_A							,      **_A	)

										UpperCAmelCase     	=       tokenizer_r.encode(_A							,      add_special_tokens=_A	)
										UpperCAmelCase     	=       tokenizer_p.encode(_A							,      add_special_tokens=_A	)

										UpperCAmelCase     	=       tokenizer_r.convert_ids_to_tokens(_A	)
										UpperCAmelCase     	=       tokenizer_p.convert_ids_to_tokens(_A	)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase     	=       [
										    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(_A	)
										]
										self.assertListEqual(_A							,      _A	)
										self.assertListEqual(_A							,      _A	)










from __future__ import annotations





def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__				)						->       bool:







			'''simple docstring'''

			UpperCAmelCase     	=       42
			UpperCAmelCase     	=       list(UpperCamelCase__				)
			UpperCAmelCase	,    UpperCAmelCase     	=       _vector[:-1], int(_vector[-1]				)
			UpperCAmelCase     	=       [int(UpperCamelCase__				) for digit in __vector]

			vector.reverse()
			for i, digit in enumerate(UpperCamelCase__				):
						if i & 1 == 0:
									UpperCAmelCase     	=       digit * 2
									if doubled > 9:
												doubled -= 9
									check_digit += doubled
						else:
									check_digit += digit

			return check_digit % 10 == 0


if __name__ == "__main__":
				import doctest

				doctest.testmod()
				assert is_luhn("79927398713")
				assert not is_luhn("79927398714")



