code: 
  
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  @require_tokenizers
  class 					A_   (      A__ ,     unittest.TestCase						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       ElectraTokenizer
      a__     							=       ElectraTokenizerFast
      a__     							=       True
      a__     							=       True
      a__     							=       filter_non_english
  
  
      def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
          super().setUp()
  
          __UpperCAmelCase									=			[
              '''[UNK]''',
              '''[CLS]''',
              '''[SEP]''',
              '''[PAD]''',
              '''[MASK]''',
              '''want''',
              '''##want''',
              '''##ed''',
              '''wa''',
              '''un''',
              '''runn''',
              '''##ing''',
              ''',''',
              '''low''',
              '''lowest''',
          ]
          __UpperCAmelCase									=			os.path.join(self.tmpdirname     ,      VOCAB_FILES_NAMES['''vocab_file''']  )
          with open(self.vocab_file     ,      '''w'''     ,      encoding='''utf-8'''  ) as vocab_writer:
              vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      List[str]:
          __UpperCAmelCase									=			'''UNwant\u00E9d,running'''
          __UpperCAmelCase									=			'''unwanted, running'''
          return input_text, output_text
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          __UpperCAmelCase									=			self.tokenizer_class(self.vocab_file  )
  
          __UpperCAmelCase									=			tokenizer.tokenize('''UNwant\u00E9d,running'''  )
          self.assertListEqual(_lowerCAmelCase     ,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase  )     ,      [9, 6, 7, 12, 10, 11]  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Dict:
          if not self.test_rust_tokenizer:
              return
  
          __UpperCAmelCase									=			self.get_tokenizer()
          __UpperCAmelCase									=			self.get_rust_tokenizer()
  
          __UpperCAmelCase									=			'''UNwant\u00E9d,running'''
  
          __UpperCAmelCase									=			tokenizer.tokenize(_lowerCAmelCase  )
          __UpperCAmelCase									=			rust_tokenizer.tokenize(_lowerCAmelCase  )
          self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
          __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
          __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
          self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
          __UpperCAmelCase									=			self.get_rust_tokenizer()
          __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase  )
          __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase  )
          self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
          # With lower casing
          __UpperCAmelCase									=			self.get_tokenizer(do_lower_case=_lowerCAmelCase  )
          __UpperCAmelCase									=			self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase  )
  
          __UpperCAmelCase									=			'''UNwant\u00E9d,running'''
  
          __UpperCAmelCase									=			tokenizer.tokenize(_lowerCAmelCase  )
          __UpperCAmelCase									=			rust_tokenizer.tokenize(_lowerCAmelCase  )
          self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
          __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
          __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
          self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
          __UpperCAmelCase									=			self.get_rust_tokenizer()
          __UpperCAmelCase									=			tokenizer.encode(_lowerCAmelCase  )
          __UpperCAmelCase									=			rust_tokenizer.encode(_lowerCAmelCase  )
          self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          __UpperCAmelCase									=			BasicTokenizer()
  
          self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )     ,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Tuple:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )     ,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
          self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''hello''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Any:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
          self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''h\u00E9llo''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      List[Any]:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
          self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''hello''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
          self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )     ,      ['''hello''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )     ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      strip_accents=_lowerCAmelCase  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )     ,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          __UpperCAmelCase									=			BasicTokenizer(do_lower_case=_lowerCAmelCase     ,      never_split=['''[UNK]''']  )
  
          self.assertListEqual(
              tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )     ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      List[str]:
          __UpperCAmelCase									=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
          __UpperCAmelCase									=			{}
          for i, token in enumerate(_lowerCAmelCase  ):
              __UpperCAmelCase									=			i
          __UpperCAmelCase									=			WordpieceTokenizer(vocab=_lowerCAmelCase     ,      unk_token='''[UNK]'''  )
  
          self.assertListEqual(tokenizer.tokenize(''''''  )     ,      []  )
  
          self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )     ,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )
  
          self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )     ,      ['''[UNK]''', '''runn''', '''##ing''']  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          self.assertTrue(_is_whitespace(''' '''  )  )
          self.assertTrue(_is_whitespace('''\t'''  )  )
          self.assertTrue(_is_whitespace('''\r'''  )  )
          self.assertTrue(_is_whitespace('''\n'''  )  )
          self.assertTrue(_is_whitespace('''\u00A0'''  )  )
  
          self.assertFalse(_is_whitespace('''A'''  )  )
          self.assertFalse(_is_whitespace('''-'''  )  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Any:
          self.assertTrue(_is_control('''\u0005'''  )  )
  
          self.assertFalse(_is_control('''A'''  )  )
          self.assertFalse(_is_control(''' '''  )  )
          self.assertFalse(_is_control('''\t'''  )  )
          self.assertFalse(_is_control('''\r'''  )  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          self.assertTrue(_is_punctuation('''-'''  )  )
          self.assertTrue(_is_punctuation('''$'''  )  )
          self.assertTrue(_is_punctuation('''`'''  )  )
          self.assertTrue(_is_punctuation('''.'''  )  )
  
          self.assertFalse(_is_punctuation('''A'''  )  )
          self.assertFalse(_is_punctuation(''' '''  )  )
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          __UpperCAmelCase									=			self.get_tokenizer()
          __UpperCAmelCase									=			self.get_rust_tokenizer()
  
          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase  ) for t in ['''Test''', '''\xad''', '''test''']]     ,      [['''[UNK]'''], [], ['''[UNK]''']]  )
  
          self.assertListEqual(
              [rust_tokenizer.tokenize(_lowerCAmelCase  ) for t in ['''Test''', '''\xad''', '''test''']]     ,      [['''[UNK]'''], [], ['''[UNK]''']]  )
  
  
      @slow
      def      lowerCAmelCase_						(self  )     ->      List[Any]:
          __UpperCAmelCase									=			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''  )
  
          __UpperCAmelCase									=			tokenizer.encode('''sequence builders'''     ,      add_special_tokens=_lowerCAmelCase  )
          __UpperCAmelCase									=			tokenizer.encode('''multi-sequence build'''     ,      add_special_tokens=_lowerCAmelCase  )
  
          __UpperCAmelCase									=			tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  )
          __UpperCAmelCase									=			tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
          assert encoded_sentence == [101] + text + [102]
          assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
      def      lowerCAmelCase_						(self  )     ->      Any:
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                  __UpperCAmelCase									=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
  
                  __UpperCAmelCase									=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                  __UpperCAmelCase									=			tokenizer_r.encode_plus(
                      _lowerCAmelCase     ,      return_attention_mask=_lowerCAmelCase     ,      return_token_type_ids=_lowerCAmelCase     ,      return_offsets_mapping=_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase     ,      )
  
                  __UpperCAmelCase									=			tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase     ,      '''do_lower_case'''  ) else False
                  __UpperCAmelCase									=			(
                      [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), '''A'''),
                          ((1, 2), ''','''),
                          ((3, 5), '''na'''),
                          ((5, 6), '''##ï'''),
                          ((6, 8), '''##ve'''),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), '''Allen'''),
                          ((21, 23), '''##NL'''),
                          ((23, 24), '''##P'''),
                          ((25, 33), '''sentence'''),
                          ((33, 34), '''.'''),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                      if not do_lower_case
                      else [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), '''a'''),
                          ((1, 2), ''','''),
                          ((3, 8), '''naive'''),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), '''allen'''),
                          ((21, 23), '''##nl'''),
                          ((23, 24), '''##p'''),
                          ((25, 33), '''sentence'''),
                          ((33, 34), '''.'''),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                  )
  
                  self.assertEqual(
                      [e[1] for e in expected_results]     ,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
                  self.assertEqual([e[0] for e in expected_results]     ,      tokens['''offset_mapping''']  )
  
  
  
  
  
  
      def      lowerCAmelCase_						(self  )     ->      Any:
          __UpperCAmelCase									=			['''的''', '''人''', '''有''']
          __UpperCAmelCase									=			''''''.join(_lowerCAmelCase  )
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                  __UpperCAmelCase									=			True
                  __UpperCAmelCase									=			self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
                  __UpperCAmelCase									=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
  
                  __UpperCAmelCase									=			tokenizer_p.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
                  __UpperCAmelCase									=			tokenizer_r.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
  
                  __UpperCAmelCase									=			tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
                  __UpperCAmelCase									=			tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )
  
                  # it is expected that each Chinese character is not preceded by "##"
                  self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
                  self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
                  __UpperCAmelCase									=			False
                  __UpperCAmelCase									=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
                  __UpperCAmelCase									=			self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,      **_lowerCAmelCase  )
  
                  __UpperCAmelCase									=			tokenizer_r.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
                  __UpperCAmelCase									=			tokenizer_p.encode(_lowerCAmelCase     ,      add_special_tokens=_lowerCAmelCase  )
  
                  __UpperCAmelCase									=			tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
                  __UpperCAmelCase									=			tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )
  
                  # it is expected that only the first Chinese character is not preceded by "##".
                  __UpperCAmelCase									=			[
                      F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase  )
                  ]
                  self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
                  self.assertListEqual(_lowerCAmelCase     ,      _lowerCAmelCase  )
  
  
  
  
style_context: 
  
  import numpy as np
  
  
  def       __a       (   SCREAMING_SNAKE_CASE      )      ->     Tuple:
  
  
  
  
  
  
      '''simple docstring'''
      __UpperCAmelCase					,     __UpperCAmelCase									=			a.shape
      __UpperCAmelCase									=			min(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )
      __UpperCAmelCase									=			np.eye(SCREAMING_SNAKE_CASE      )
      __UpperCAmelCase									=			a.copy()
  
      for k in range(t - 1      ):
          # select a column of modified matrix A':
          __UpperCAmelCase									=			r[k:, [k]]
          # construct first basis vector
          __UpperCAmelCase									=			np.zeros_like(SCREAMING_SNAKE_CASE      )
          __UpperCAmelCase									=			1.0
          # determine scaling factor
          __UpperCAmelCase									=			np.linalg.norm(SCREAMING_SNAKE_CASE      )
          # construct vector v for Householder reflection
          __UpperCAmelCase									=			x + np.sign(x[0]      ) * alpha * ea
          v /= np.linalg.norm(SCREAMING_SNAKE_CASE      )
  
          # construct the Householder matrix
          __UpperCAmelCase									=			np.eye(m - k      ) - 2.0 * v @ v.T
          # pad with ones and zeros as necessary
          __UpperCAmelCase									=			np.block([[np.eye(SCREAMING_SNAKE_CASE      ), np.zeros((k, m - k)      )], [np.zeros((m - k, k)      ), q_k]]      )
  
          __UpperCAmelCase									=			q @ q_k.T
          __UpperCAmelCase									=			q_k @ r
  
      return q, r
  
  
  if __name__ == "__main__":
         import doctest
  
         doctest.testmod()
  
  
  
  
  