





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 		_lowerCAmelCase(		_a						,       unittest.TestCase):

  """simple docstring"""







  lowerCamelCase__     	=       ElectraTokenizer
  lowerCamelCase__     	=       ElectraTokenizerFast
  lowerCamelCase__     	=       True
  lowerCamelCase__     	=       True
  lowerCamelCase__     	=       filter_non_english

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Any:
     super().setUp()

     __A    =  [
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     __A    =  os.path.join(self.tmpdirname    ,							VOCAB_FILES_NAMES['''vocab_file''']      )
     with open(self.vocab_file    ,							'''w'''    ,							encoding='''utf-8'''      ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   Optional[Any]:
     __A    =  '''UNwant\u00E9d,running'''
     __A    =  '''unwanted, running'''
     return input_text, output_text

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   str:
     __A    =  self.tokenizer_class(self.vocab_file      )

     __A    =  tokenizer.tokenize('''UNwant\u00E9d,running'''      )
     self.assertListEqual(UpperCAmelCase    ,							['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']      )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase      )    ,							[9, 6, 7, 12, 10, 11]      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[Any]:
     if not self.test_rust_tokenizer:
        return

     __A    =  self.get_tokenizer()
     __A    =  self.get_rust_tokenizer()

     __A    =  '''UNwant\u00E9d,running'''

     __A    =  tokenizer.tokenize(UpperCAmelCase      )
     __A    =  rust_tokenizer.tokenize(UpperCAmelCase      )
     self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

     __A    =  tokenizer.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )
     __A    =  rust_tokenizer.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )
     self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

     __A    =  self.get_rust_tokenizer()
     __A    =  tokenizer.encode(UpperCAmelCase      )
     __A    =  rust_tokenizer.encode(UpperCAmelCase      )
     self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

     # With lower casing
     __A    =  self.get_tokenizer(do_lower_case=UpperCAmelCase      )
     __A    =  self.get_rust_tokenizer(do_lower_case=UpperCAmelCase      )

     __A    =  '''UNwant\u00E9d,running'''

     __A    =  tokenizer.tokenize(UpperCAmelCase      )
     __A    =  rust_tokenizer.tokenize(UpperCAmelCase      )
     self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

     __A    =  tokenizer.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )
     __A    =  rust_tokenizer.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )
     self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

     __A    =  self.get_rust_tokenizer()
     __A    =  tokenizer.encode(UpperCAmelCase      )
     __A    =  rust_tokenizer.encode(UpperCAmelCase      )
     self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   str:
     __A    =  BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      )    ,							['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   List[Any]:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )    ,							['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,							['''hello''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[Any]:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase    ,							strip_accents=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,							['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,							['''h\u00E9llo''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[Any]:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase    ,							strip_accents=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,							['''hello''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Any:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,							['''hello''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[Any]:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )    ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[int]:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase    ,							strip_accents=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,							['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   List[str]:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase    ,							strip_accents=UpperCAmelCase      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,							['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Any:
     __A    =  BasicTokenizer(do_lower_case=UpperCAmelCase    ,							never_split=['''[UNK]''']      )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      )    ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   int:
     __A    =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

     __A    =  {}
     for i, token in enumerate(UpperCAmelCase      ):
        __A    =  i
     __A    =  WordpieceTokenizer(vocab=UpperCAmelCase    ,							unk_token='''[UNK]'''      )

     self.assertListEqual(tokenizer.tokenize(''''''      )    ,							[]      )

     self.assertListEqual(tokenizer.tokenize('''unwanted running'''      )    ,							['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      )    ,							['''[UNK]''', '''runn''', '''##ing''']      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Tuple:
     self.assertTrue(_is_whitespace(''' '''      )      )
     self.assertTrue(_is_whitespace('''\t'''      )      )
     self.assertTrue(_is_whitespace('''\r'''      )      )
     self.assertTrue(_is_whitespace('''\n'''      )      )
     self.assertTrue(_is_whitespace('''\u00A0'''      )      )

     self.assertFalse(_is_whitespace('''A'''      )      )
     self.assertFalse(_is_whitespace('''-'''      )      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[Any]:
     self.assertTrue(_is_control('''\u0005'''      )      )

     self.assertFalse(_is_control('''A'''      )      )
     self.assertFalse(_is_control(''' '''      )      )
     self.assertFalse(_is_control('''\t'''      )      )
     self.assertFalse(_is_control('''\r'''      )      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Union[str, Any]:
     self.assertTrue(_is_punctuation('''-'''      )      )
     self.assertTrue(_is_punctuation('''$'''      )      )
     self.assertTrue(_is_punctuation('''`'''      )      )
     self.assertTrue(_is_punctuation('''.'''      )      )

     self.assertFalse(_is_punctuation('''A'''      )      )
     self.assertFalse(_is_punctuation(''' '''      )      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Tuple:
     __A    =  self.get_tokenizer()
     __A    =  self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(UpperCAmelCase      ) for t in ['''Test''', '''\xad''', '''test''']]    ,							[['''[UNK]'''], [], ['''[UNK]''']]      )

     self.assertListEqual(
         [rust_tokenizer.tokenize(UpperCAmelCase      ) for t in ['''Test''', '''\xad''', '''test''']]    ,							[['''[UNK]'''], [], ['''[UNK]''']]      )

  @slow
  def 						SCREAMING_SNAKE_CASE__			(   self      )->   int:
     __A    =  self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''      )

     __A    =  tokenizer.encode('''sequence builders'''    ,							add_special_tokens=UpperCAmelCase      )
     __A    =  tokenizer.encode('''multi-sequence build'''    ,							add_special_tokens=UpperCAmelCase      )

     __A    =  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase      )
     __A    =  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase    ,							UpperCAmelCase      )

     assert encoded_sentence == [1_01] + text + [1_02]
     assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   List[str]:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
           __A    =  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase    ,							**UpperCAmelCase      )

           __A    =  f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
           __A    =  tokenizer_r.encode_plus(
               UpperCAmelCase    ,							return_attention_mask=UpperCAmelCase    ,							return_token_type_ids=UpperCAmelCase    ,							return_offsets_mapping=UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase    ,							)

           __A    =  tokenizer_r.do_lower_case if hasattr(UpperCAmelCase    ,							'''do_lower_case'''      ) else False
           __A    =  (
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''A'''),
                   ((1, 2), ''','''),
                   ((3, 5), '''na'''),
                   ((5, 6), '''##ï'''),
                   ((6, 8), '''##ve'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''Allen'''),
                   ((21, 23), '''##NL'''),
                   ((23, 24), '''##P'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''a'''),
                   ((1, 2), ''','''),
                   ((3, 8), '''naive'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''allen'''),
                   ((21, 23), '''##nl'''),
                   ((23, 24), '''##p'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]    ,							tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
           self.assertEqual([e[0] for e in expected_results]    ,							tokens['''offset_mapping''']      )





  def 						SCREAMING_SNAKE_CASE__			(   self      )->   List[str]:
     __A    =  ['''的''', '''人''', '''有''']
     __A    =  ''''''.join(UpperCAmelCase      )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
           __A    =  True
           __A    =  self.tokenizer_class.from_pretrained(UpperCAmelCase    ,							**UpperCAmelCase      )
           __A    =  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase    ,							**UpperCAmelCase      )

           __A    =  tokenizer_p.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )
           __A    =  tokenizer_r.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )

           __A    =  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase      )
           __A    =  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase      )

           # it is expected that each Chinese character is not preceded by "##"
           self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )
           self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )

           __A    =  False
           __A    =  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase    ,							**UpperCAmelCase      )
           __A    =  self.tokenizer_class.from_pretrained(UpperCAmelCase    ,							**UpperCAmelCase      )

           __A    =  tokenizer_r.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )
           __A    =  tokenizer_p.encode(UpperCAmelCase    ,							add_special_tokens=UpperCAmelCase      )

           __A    =  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase      )
           __A    =  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase      )

           # it is expected that only the first Chinese character is not preceded by "##".
           __A    =  [
               f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase      )
           ]
           self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )
           self.assertListEqual(UpperCAmelCase    ,							UpperCAmelCase      )








import warnings

from ...utils import logging
from .image_processing_detr import DetrImageProcessor


_UpperCamelCase       :				Tuple           =			logging.get_logger(__name__)







class 		_lowerCAmelCase(		_a):

  """simple docstring"""







  def __init__(   self    ,							*UpperCAmelCase    ,							**UpperCAmelCase      )->   None:
     warnings.warn(
         '''The class DetrFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'''
         ''' Please use DetrImageProcessor instead.'''    ,							UpperCAmelCase    ,							)
     super().__init__(*UpperCAmelCase    ,							**UpperCAmelCase      )

