code: 
  
  
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  @require_tokenizers
  class 						A__      (					snake_case__ ,      unittest.TestCase			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      __magic_name__    						=	ElectraTokenizer
      __magic_name__    						=	ElectraTokenizerFast
      __magic_name__    						=	True
      __magic_name__    						=	True
      __magic_name__    						=	filter_non_english
      def 	a_    ( self					):
           super().setUp()
  
           snake_case					=       [
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           snake_case					=       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''vocab_file''']					)
           with open(self.vocab_file    ,      '''w'''    ,      encoding='''utf-8'''					) as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
      def 	a_    ( self    ,      __snake_case					):
           snake_case					=       '''UNwant\u00E9d,running'''
           snake_case					=       '''unwanted, running'''
           return input_text, output_text
      def 	a_    ( self					):
           snake_case					=       self.tokenizer_class(self.vocab_file					)
  
           snake_case					=       tokenizer.tokenize('''UNwant\u00E9d,running'''					)
           self.assertListEqual(__snake_case    ,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case					)    ,      [9, 6, 7, 1_2, 1_0, 1_1]					)
      def 	a_    ( self					):
           if not self.test_rust_tokenizer:
                return
  
           snake_case					=       self.get_tokenizer()
           snake_case					=       self.get_rust_tokenizer()
  
           snake_case					=       '''UNwant\u00E9d,running'''
  
           snake_case					=       tokenizer.tokenize(__snake_case					)
           snake_case					=       rust_tokenizer.tokenize(__snake_case					)
           self.assertListEqual(__snake_case    ,      __snake_case					)
  
           snake_case					=       tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
           snake_case					=       rust_tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
           self.assertListEqual(__snake_case    ,      __snake_case					)
  
           snake_case					=       self.get_rust_tokenizer()
           snake_case					=       tokenizer.encode(__snake_case					)
           snake_case					=       rust_tokenizer.encode(__snake_case					)
           self.assertListEqual(__snake_case    ,      __snake_case					)
  
           # With lower casing
           snake_case					=       self.get_tokenizer(do_lower_case=__snake_case					)
           snake_case					=       self.get_rust_tokenizer(do_lower_case=__snake_case					)
  
           snake_case					=       '''UNwant\u00E9d,running'''
  
           snake_case					=       tokenizer.tokenize(__snake_case					)
           snake_case					=       rust_tokenizer.tokenize(__snake_case					)
           self.assertListEqual(__snake_case    ,      __snake_case					)
  
           snake_case					=       tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
           snake_case					=       rust_tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
           self.assertListEqual(__snake_case    ,      __snake_case					)
  
           snake_case					=       self.get_rust_tokenizer()
           snake_case					=       tokenizer.encode(__snake_case					)
           snake_case					=       rust_tokenizer.encode(__snake_case					)
           self.assertListEqual(__snake_case    ,      __snake_case					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer()
  
           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)    ,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)    ,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''hello''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''h\u00E9llo''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''hello''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''hello''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)    ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
      def 	a_    ( self					):
           snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      never_split=['''[UNK]''']					)
  
           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)    ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
      def 	a_    ( self					):
           snake_case					=       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
           snake_case					=       {}
           for i, token in enumerate(__snake_case					):
                snake_case					=       i
           snake_case					=       WordpieceTokenizer(vocab=__snake_case    ,      unk_token='''[UNK]'''					)
  
           self.assertListEqual(tokenizer.tokenize(''''''					)    ,      []					)
  
           self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)    ,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)
  
           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)    ,      ['''[UNK]''', '''runn''', '''##ing''']					)
      def 	a_    ( self					):
           self.assertTrue(_is_whitespace(''' '''					)					)
           self.assertTrue(_is_whitespace('''\t'''					)					)
           self.assertTrue(_is_whitespace('''\r'''					)					)
           self.assertTrue(_is_whitespace('''\n'''					)					)
           self.assertTrue(_is_whitespace('''\u00A0'''					)					)
  
           self.assertFalse(_is_whitespace('''A'''					)					)
           self.assertFalse(_is_whitespace('''-'''					)					)
      def 	a_    ( self					):
           self.assertTrue(_is_control('''\u0005'''					)					)
  
           self.assertFalse(_is_control('''A'''					)					)
           self.assertFalse(_is_control(''' '''					)					)
           self.assertFalse(_is_control('''\t'''					)					)
           self.assertFalse(_is_control('''\r'''					)					)
      def 	a_    ( self					):
           self.assertTrue(_is_punctuation('''-'''					)					)
           self.assertTrue(_is_punctuation('''$'''					)					)
           self.assertTrue(_is_punctuation('''`'''					)					)
           self.assertTrue(_is_punctuation('''.'''					)					)
  
           self.assertFalse(_is_punctuation('''A'''					)					)
           self.assertFalse(_is_punctuation(''' '''					)					)
      def 	a_    ( self					):
           snake_case					=       self.get_tokenizer()
           snake_case					=       self.get_rust_tokenizer()
  
           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(__snake_case					) for t in ['''Test''', '''\xad''', '''test''']]    ,      [['''[UNK]'''], [], ['''[UNK]''']]					)
  
           self.assertListEqual(
               [rust_tokenizer.tokenize(__snake_case					) for t in ['''Test''', '''\xad''', '''test''']]    ,      [['''[UNK]'''], [], ['''[UNK]''']]					)
      @slow
      def 	a_    ( self					):
           snake_case					=       self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''					)
  
           snake_case					=       tokenizer.encode('''sequence builders'''    ,      add_special_tokens=__snake_case					)
           snake_case					=       tokenizer.encode('''multi-sequence build'''    ,      add_special_tokens=__snake_case					)
  
           snake_case					=       tokenizer.build_inputs_with_special_tokens(__snake_case					)
           snake_case					=       tokenizer.build_inputs_with_special_tokens(__snake_case    ,      __snake_case					)
  
           assert encoded_sentence == [1_0_1] + text + [1_0_2]
           assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
      def 	a_    ( self					):
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                     snake_case					=       self.rust_tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
  
                     snake_case					=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                     snake_case					=       tokenizer_r.encode_plus(
                         __snake_case    ,      return_attention_mask=__snake_case    ,      return_token_type_ids=__snake_case    ,      return_offsets_mapping=__snake_case    ,      add_special_tokens=__snake_case    ,      )
  
                     snake_case					=       tokenizer_r.do_lower_case if hasattr(__snake_case    ,      '''do_lower_case'''					) else False
                     snake_case					=       (
                         [
                             ((0, 0), tokenizer_r.cls_token),
                             ((0, 1), '''A'''),
                             ((1, 2), ''','''),
                             ((3, 5), '''na'''),
                             ((5, 6), '''##ï'''),
                             ((6, 8), '''##ve'''),
                             ((9, 1_5), tokenizer_r.mask_token),
                             ((1_6, 2_1), '''Allen'''),
                             ((2_1, 2_3), '''##NL'''),
                             ((2_3, 2_4), '''##P'''),
                             ((2_5, 3_3), '''sentence'''),
                             ((3_3, 3_4), '''.'''),
                             ((0, 0), tokenizer_r.sep_token),
                         ]
                         if not do_lower_case
                         else [
                             ((0, 0), tokenizer_r.cls_token),
                             ((0, 1), '''a'''),
                             ((1, 2), ''','''),
                             ((3, 8), '''naive'''),
                             ((9, 1_5), tokenizer_r.mask_token),
                             ((1_6, 2_1), '''allen'''),
                             ((2_1, 2_3), '''##nl'''),
                             ((2_3, 2_4), '''##p'''),
                             ((2_5, 3_3), '''sentence'''),
                             ((3_3, 3_4), '''.'''),
                             ((0, 0), tokenizer_r.sep_token),
                         ]
                     )
  
                     self.assertEqual(
                         [e[1] for e in expected_results]    ,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
                     self.assertEqual([e[0] for e in expected_results]    ,      tokens['''offset_mapping''']					)
  
  
  
  
      def 	a_    ( self					):
           snake_case					=       ['''的''', '''人''', '''有''']
           snake_case					=       ''''''.join(__snake_case					)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                     snake_case					=       True
                     snake_case					=       self.tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
                     snake_case					=       self.rust_tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
  
                     snake_case					=       tokenizer_p.encode(__snake_case    ,      add_special_tokens=__snake_case					)
                     snake_case					=       tokenizer_r.encode(__snake_case    ,      add_special_tokens=__snake_case					)
  
                     snake_case					=       tokenizer_r.convert_ids_to_tokens(__snake_case					)
                     snake_case					=       tokenizer_p.convert_ids_to_tokens(__snake_case					)
  
                     # it is expected that each Chinese character is not preceded by "##"
                     self.assertListEqual(__snake_case    ,      __snake_case					)
                     self.assertListEqual(__snake_case    ,      __snake_case					)
  
                     snake_case					=       False
                     snake_case					=       self.rust_tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
                     snake_case					=       self.tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
  
                     snake_case					=       tokenizer_r.encode(__snake_case    ,      add_special_tokens=__snake_case					)
                     snake_case					=       tokenizer_p.encode(__snake_case    ,      add_special_tokens=__snake_case					)
  
                     snake_case					=       tokenizer_r.convert_ids_to_tokens(__snake_case					)
                     snake_case					=       tokenizer_p.convert_ids_to_tokens(__snake_case					)
  
                     # it is expected that only the first Chinese character is not preceded by "##".
                     snake_case					=       [
                         F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__snake_case					)
                     ]
                     self.assertListEqual(__snake_case    ,      __snake_case					)
                     self.assertListEqual(__snake_case    ,      __snake_case					)
  
  
  
  
  
style_context: 
  
  
  from manim import *
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def 	a_    ( self					):
           snake_case					=       Rectangle(height=0.5    ,      width=0.5					)
           snake_case					=       Rectangle(height=0.46    ,      width=0.46					).set_stroke(width=0					)
           snake_case					=       Rectangle(height=0.25    ,      width=0.25					)
  
           snake_case					=       [mem.copy() for i in range(6					)]
           snake_case					=       [mem.copy() for i in range(6					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       Text('''CPU'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           cpu.move_to([-2.5, -0.5, 0]					)
           self.add(__snake_case					)
  
           snake_case					=       [mem.copy() for i in range(4					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       Text('''GPU'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           gpu.move_to([-1, -1, 0]					)
           self.add(__snake_case					)
  
           snake_case					=       [mem.copy() for i in range(6					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
  
           snake_case					=       Text('''Model'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           model.move_to([3, -1.0, 0]					)
           self.add(__snake_case					)
  
           snake_case					=       []
           snake_case					=       []
  
           for i, rect in enumerate(__snake_case					):
                rect.set_stroke(__snake_case					)
  
                snake_case					=       Rectangle(height=0.46 / 4    ,      width=0.46 / 3					).set_stroke(width=0.0					).set_fill(__snake_case    ,      opacity=0.7					)
  
                if i == 0:
                     cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)    ,      buff=0.02    ,      direction=__snake_case					)
                     cpu_target.set_x(cpu_target.get_x() + 0.1					)
                elif i == 3:
                     cpu_target.next_to(model_cpu_arr[0]    ,      direction=__snake_case    ,      buff=0.0					)
                else:
                     cpu_target.next_to(model_cpu_arr[i - 1]    ,      direction=__snake_case    ,      buff=0.0					)
                self.add(__snake_case					)
                model_cpu_arr.append(__snake_case					)
  
           self.add(*__snake_case    ,      *__snake_case					)
  
           snake_case					=       [meta_mem.copy() for i in range(6					)]
           snake_case					=       [meta_mem.copy() for i in range(6					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       Text('''Disk'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           disk.move_to([-4.0, -1.25, 0]					)
           self.add(__snake_case    ,      __snake_case					)
  
           snake_case					=       []
  
           for i in range(6					):
                snake_case					=       fill.copy().set_fill(__snake_case    ,      opacity=0.8					)
                target.move_to(disk_left_col_base[i]					).scale(0.5					)
                cpu_disk_arr.append(__snake_case					)
  
           self.add(*__snake_case					)
  
           snake_case					=       Square(side_length=2.2					)
           key.move_to([-5, 2, 0]					)
  
           snake_case					=       MarkupText(
               F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''    ,      font_size=1_8    ,      )
  
           key_text.move_to([-5, 2.4, 0]					)
  
           self.add(__snake_case    ,      __snake_case					)
  
           snake_case					=       MarkupText(
               F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''    ,      font_size=1_8    ,      )
  
           blue_text.next_to(__snake_case    ,      DOWN * 2.4    ,      aligned_edge=key_text.get_left()					)
           self.add(__snake_case					)
  
           snake_case					=       MarkupText(F'''The offloaded weights are all sent to the CPU.'''    ,      font_size=2_4					)
           step_a.move_to([2, 2, 0]					)
  
           self.play(Write(__snake_case    ,      run_time=3					)					)
  
           for i in range(6					):
                snake_case					=       cpu_disk_arr[i]
                snake_case					=       rect.copy().set_fill(__snake_case    ,      opacity=0.8					).scale(2.0					)
                cpa.generate_target()
                cpa.target.move_to(model_base[i]					)
  
                if i == 0:
                     rect.set_fill(__snake_case    ,      opacity=0.8					)
                     rect.generate_target()
                     rect.target.move_to(cpu_left_col_base[0]					).scale(2.0					)
  
                     self.remove(
                         *__snake_case    ,      *__snake_case    ,      )
  
                else:
                     rect.generate_target()
                     rect.target.move_to(cpu_left_col_base[i]					).scale(2.0					)
                self.play(MoveToTarget(__snake_case					)    ,      MoveToTarget(__snake_case					)    ,      model_base[i].animate.set_stroke(__snake_case					)					)
           self.play(FadeOut(__snake_case					)					)
  
           snake_case					=       MarkupText(
               F'''Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.'''    ,      font_size=2_4    ,      )
           step_a.move_to([2, 2, 0]					)
  
           self.play(Write(__snake_case    ,      run_time=3					)					)
  
           snake_case					=       []
           snake_case					=       []
           for i in range(6					):
                snake_case					=       Arrow(start=__snake_case    ,      end=__snake_case    ,      color=__snake_case    ,      buff=0.5					)
                a.next_to(model_base[i].get_left()    ,      __snake_case    ,      buff=0.2					)
                arrows.append(__snake_case					)
                animations.append(Write(__snake_case					)					)
           self.play(*__snake_case					)
           self.wait()
  
  
  
  
  