code: 
  """simple docstring"""
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  @require_tokenizers
  class    _lowerCAmelCase						(						lowerCamelCase    ,       unittest.TestCase						):
         lowercase_     :     Optional[int]       =							ElectraTokenizer
         lowercase_     :     List[Any]       =							ElectraTokenizerFast
         lowercase_     :     List[str]       =							True
         lowercase_     :     Union[str, Any]       =							True
         lowercase_     :     int       =							filter_non_english
  
  
  
         def       _a   (      self   )							->							str:
          super().setUp()
  
          _UpperCAmelCase =						[
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          _UpperCAmelCase =						os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES["vocab_file"]   )
          with open(self.vocab_file ,						"w" ,						encoding="utf-8"   ) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )
  
  
  
         def       _a   (      self ,						a_   )							->							List[str]:
          _UpperCAmelCase =						"UNwant\u00E9d,running"
          _UpperCAmelCase =						"unwanted, running"
          return input_text, output_text
  
  
  
         def       _a   (      self   )							->							Any:
          _UpperCAmelCase =						self.tokenizer_class(self.vocab_file   )
  
          _UpperCAmelCase =						tokenizer.tokenize("UNwant\u00E9d,running"   )
          self.assertListEqual(a_ ,						["un", "##want", "##ed", ",", "runn", "##ing"]   )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(a_   ) ,						[9, 6, 7, 12, 10, 11]   )
  
  
  
         def       _a   (      self   )							->							Dict:
          if not self.test_rust_tokenizer:
           return
  
          _UpperCAmelCase =						self.get_tokenizer()
          _UpperCAmelCase =						self.get_rust_tokenizer()
  
          _UpperCAmelCase =						"UNwant\u00E9d,running"
  
          _UpperCAmelCase =						tokenizer.tokenize(a_   )
          _UpperCAmelCase =						rust_tokenizer.tokenize(a_   )
          self.assertListEqual(a_ ,						a_   )
  
          _UpperCAmelCase =						tokenizer.encode(a_ ,						add_special_tokens=a_   )
          _UpperCAmelCase =						rust_tokenizer.encode(a_ ,						add_special_tokens=a_   )
          self.assertListEqual(a_ ,						a_   )
  
          _UpperCAmelCase =						self.get_rust_tokenizer()
          _UpperCAmelCase =						tokenizer.encode(a_   )
          _UpperCAmelCase =						rust_tokenizer.encode(a_   )
          self.assertListEqual(a_ ,						a_   )
  
          # With lower casing
          _UpperCAmelCase =						self.get_tokenizer(do_lower_case=a_   )
          _UpperCAmelCase =						self.get_rust_tokenizer(do_lower_case=a_   )
  
          _UpperCAmelCase =						"UNwant\u00E9d,running"
  
          _UpperCAmelCase =						tokenizer.tokenize(a_   )
          _UpperCAmelCase =						rust_tokenizer.tokenize(a_   )
          self.assertListEqual(a_ ,						a_   )
  
          _UpperCAmelCase =						tokenizer.encode(a_ ,						add_special_tokens=a_   )
          _UpperCAmelCase =						rust_tokenizer.encode(a_ ,						add_special_tokens=a_   )
          self.assertListEqual(a_ ,						a_   )
  
          _UpperCAmelCase =						self.get_rust_tokenizer()
          _UpperCAmelCase =						tokenizer.encode(a_   )
          _UpperCAmelCase =						rust_tokenizer.encode(a_   )
          self.assertListEqual(a_ ,						a_   )
  
  
  
         def       _a   (      self   )							->							int:
          _UpperCAmelCase =						BasicTokenizer()
  
          self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"   ) ,						["ah", "\u535A", "\u63A8", "zz"]   )
  
  
  
         def       _a   (      self   )							->							List[Any]:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   ) ,						["hello", "!", "how", "are", "you", "?"]   )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ) ,						["hello"]   )
  
  
  
         def       _a   (      self   )							->							Optional[int]:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_ ,						strip_accents=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ) ,						["hällo", "!", "how", "are", "you", "?"]   )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ) ,						["h\u00E9llo"]   )
  
  
  
         def       _a   (      self   )							->							int:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_ ,						strip_accents=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ) ,						["hallo", "!", "how", "are", "you", "?"]   )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ) ,						["hello"]   )
  
  
  
         def       _a   (      self   )							->							Any:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ) ,						["hallo", "!", "how", "are", "you", "?"]   )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ) ,						["hello"]   )
  
  
  
         def       _a   (      self   )							->							List[str]:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   ) ,						["HeLLo", "!", "how", "Are", "yoU", "?"]   )
  
  
  
         def       _a   (      self   )							->							List[str]:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_ ,						strip_accents=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ) ,						["HäLLo", "!", "how", "Are", "yoU", "?"]   )
  
  
  
         def       _a   (      self   )							->							Union[str, Any]:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_ ,						strip_accents=a_   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ) ,						["HaLLo", "!", "how", "Are", "yoU", "?"]   )
  
  
  
         def       _a   (      self   )							->							Any:
          _UpperCAmelCase =						BasicTokenizer(do_lower_case=a_ ,						never_split=["[UNK]"]   )
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"   ) ,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]   )
  
  
  
         def       _a   (      self   )							->							List[str]:
          _UpperCAmelCase =						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
          _UpperCAmelCase =						{}
          for i, token in enumerate(a_   ):
           _UpperCAmelCase =						i
          _UpperCAmelCase =						WordpieceTokenizer(vocab=a_ ,						unk_token="[UNK]"   )
  
          self.assertListEqual(tokenizer.tokenize(""   ) ,						[]   )
  
          self.assertListEqual(tokenizer.tokenize("unwanted running"   ) ,						["un", "##want", "##ed", "runn", "##ing"]   )
  
          self.assertListEqual(tokenizer.tokenize("unwantedX running"   ) ,						["[UNK]", "runn", "##ing"]   )
  
  
  
         def       _a   (      self   )							->							List[str]:
          self.assertTrue(_is_whitespace(" "   )   )
          self.assertTrue(_is_whitespace("\t"   )   )
          self.assertTrue(_is_whitespace("\r"   )   )
          self.assertTrue(_is_whitespace("\n"   )   )
          self.assertTrue(_is_whitespace("\u00A0"   )   )
  
          self.assertFalse(_is_whitespace("A"   )   )
          self.assertFalse(_is_whitespace("-"   )   )
  
  
  
         def       _a   (      self   )							->							int:
          self.assertTrue(_is_control("\u0005"   )   )
  
          self.assertFalse(_is_control("A"   )   )
          self.assertFalse(_is_control(" "   )   )
          self.assertFalse(_is_control("\t"   )   )
          self.assertFalse(_is_control("\r"   )   )
  
  
  
         def       _a   (      self   )							->							Tuple:
          self.assertTrue(_is_punctuation("-"   )   )
          self.assertTrue(_is_punctuation("$"   )   )
          self.assertTrue(_is_punctuation("`"   )   )
          self.assertTrue(_is_punctuation("."   )   )
  
          self.assertFalse(_is_punctuation("A"   )   )
          self.assertFalse(_is_punctuation(" "   )   )
  
  
  
         def       _a   (      self   )							->							Optional[Any]:
          _UpperCAmelCase =						self.get_tokenizer()
          _UpperCAmelCase =						self.get_rust_tokenizer()
  
          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(a_   ) for t in ["Test", "\xad", "test"]] ,						[["[UNK]"], [], ["[UNK]"]]   )
  
          self.assertListEqual(
              [rust_tokenizer.tokenize(a_   ) for t in ["Test", "\xad", "test"]] ,						[["[UNK]"], [], ["[UNK]"]]   )
  
  
  
         @slow
         def       _a   (      self   )							->							Tuple:
          _UpperCAmelCase =						self.tokenizer_class.from_pretrained("google/electra-base-discriminator"   )
  
          _UpperCAmelCase =						tokenizer.encode("sequence builders" ,						add_special_tokens=a_   )
          _UpperCAmelCase =						tokenizer.encode("multi-sequence build" ,						add_special_tokens=a_   )
  
          _UpperCAmelCase =						tokenizer.build_inputs_with_special_tokens(a_   )
          _UpperCAmelCase =						tokenizer.build_inputs_with_special_tokens(a_ ,						a_   )
  
          assert encoded_sentence == [101] + text + [102]
          assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
         def       _a   (      self   )							->							Optional[Any]:
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
            _UpperCAmelCase =						self.rust_tokenizer_class.from_pretrained(a_ ,						**a_   )
  
            _UpperCAmelCase =						f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
            _UpperCAmelCase =						tokenizer_r.encode_plus(
                a_ ,						return_attention_mask=a_ ,						return_token_type_ids=a_ ,						return_offsets_mapping=a_ ,						add_special_tokens=a_ ,						)
  
            _UpperCAmelCase =						tokenizer_r.do_lower_case if hasattr(a_ ,						"do_lower_case"   ) else False
            _UpperCAmelCase =						(
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "A"),
                    ((1, 2), ","),
                    ((3, 5), "na"),
                    ((5, 6), "##ï"),
                    ((6, 8), "##ve"),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), "Allen"),
                    ((21, 23), "##NL"),
                    ((23, 24), "##P"),
                    ((25, 33), "sentence"),
                    ((33, 34), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "a"),
                    ((1, 2), ","),
                    ((3, 8), "naive"),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), "allen"),
                    ((21, 23), "##nl"),
                    ((23, 24), "##p"),
                    ((25, 33), "sentence"),
                    ((33, 34), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )
  
            self.assertEqual(
                [e[1] for e in expected_results] ,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]   )   )
            self.assertEqual([e[0] for e in expected_results] ,						tokens["offset_mapping"]   )
  
  
  
         def       _a   (      self   )							->							Any:
          _UpperCAmelCase =						["的", "人", "有"]
          _UpperCAmelCase =						"".join(a_   )
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
            _UpperCAmelCase =						True
            _UpperCAmelCase =						self.tokenizer_class.from_pretrained(a_ ,						**a_   )
            _UpperCAmelCase =						self.rust_tokenizer_class.from_pretrained(a_ ,						**a_   )
  
            _UpperCAmelCase =						tokenizer_p.encode(a_ ,						add_special_tokens=a_   )
            _UpperCAmelCase =						tokenizer_r.encode(a_ ,						add_special_tokens=a_   )
  
            _UpperCAmelCase =						tokenizer_r.convert_ids_to_tokens(a_   )
            _UpperCAmelCase =						tokenizer_p.convert_ids_to_tokens(a_   )
  
            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(a_ ,						a_   )
            self.assertListEqual(a_ ,						a_   )
  
            _UpperCAmelCase =						False
            _UpperCAmelCase =						self.rust_tokenizer_class.from_pretrained(a_ ,						**a_   )
            _UpperCAmelCase =						self.tokenizer_class.from_pretrained(a_ ,						**a_   )
  
            _UpperCAmelCase =						tokenizer_r.encode(a_ ,						add_special_tokens=a_   )
            _UpperCAmelCase =						tokenizer_p.encode(a_ ,						add_special_tokens=a_   )
  
            _UpperCAmelCase =						tokenizer_r.convert_ids_to_tokens(a_   )
            _UpperCAmelCase =						tokenizer_p.convert_ids_to_tokens(a_   )
  
            # it is expected that only the first Chinese character is not preceded by "##".
            _UpperCAmelCase =						[
                f"##{token}" if idx != 0 else token for idx, token in enumerate(a_   )
            ]
            self.assertListEqual(a_ ,						a_   )
            self.assertListEqual(a_ ,						a_   )
  
  
  
  
  
style_context: 
  """simple docstring"""
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available
  
  
  __magic_name__		       =				{}
  
  try:
  			if not is_sentencepiece_available():
  						raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  			pass
  else:
  			__magic_name__		       =				['''CpmTokenizer''']
  
  try:
  			if not is_tokenizers_available():
  						raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  			pass
  else:
  			__magic_name__		       =				['''CpmTokenizerFast''']
  
  
  if TYPE_CHECKING:
  			try:
  						if not is_sentencepiece_available():
  									raise OptionalDependencyNotAvailable()
  			except OptionalDependencyNotAvailable:
  						pass
  			else:
  						from .tokenization_cpm import CpmTokenizer
  
  			try:
  						if not is_tokenizers_available():
  									raise OptionalDependencyNotAvailable()
  			except OptionalDependencyNotAvailable:
  						pass
  			else:
  						from .tokenization_cpm_fast import CpmTokenizerFast
  
  else:
  			import sys
  
  			__magic_name__		       =				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
  
  
  
  
  