code: 
  
  
  """simple docstring"""
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  @require_tokenizers
  class   lowercase__							(				a_   ,unittest.TestCase							):
     __UpperCAmelCase       					=				ElectraTokenizer
     __UpperCAmelCase       					=				ElectraTokenizerFast
     __UpperCAmelCase       					=				True
     __UpperCAmelCase       					=				True
     __UpperCAmelCase       					=				filter_non_english
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     str:
      super().setUp()
  
      _lowerCamelCase  :				Optional[int]      =		[
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      _lowerCamelCase  :				Optional[Any]      =		os.path.join(self.tmpdirname			,      VOCAB_FILES_NAMES["""vocab_file"""])
      with open(self.vocab_file			,      """w"""			,      encoding="""utf-8""") as vocab_writer:
       vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Optional[Any]:
      _lowerCamelCase  :				int      =		'''UNwant\u00E9d,running'''
      _lowerCamelCase  :				Optional[Any]      =		'''unwanted, running'''
      return input_text, output_text
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     List[Any]:
      _lowerCamelCase  :				List[str]      =		self.tokenizer_class(self.vocab_file)
  
      _lowerCamelCase  :				Optional[int]      =		tokenizer.tokenize("""UNwant\u00E9d,running""")
      self.assertListEqual(_snake_case			,      ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case)			,      [9, 6, 7, 12, 10, 11])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Union[str, Any]:
      if not self.test_rust_tokenizer:
       return
  
      _lowerCamelCase  :				Dict      =		self.get_tokenizer()
      _lowerCamelCase  :				int      =		self.get_rust_tokenizer()
  
      _lowerCamelCase  :				Union[str, Any]      =		'''UNwant\u00E9d,running'''
  
      _lowerCamelCase  :				List[str]      =		tokenizer.tokenize(_snake_case)
      _lowerCamelCase  :				Optional[Any]      =		rust_tokenizer.tokenize(_snake_case)
      self.assertListEqual(_snake_case			,      _snake_case)
  
      _lowerCamelCase  :				Union[str, Any]      =		tokenizer.encode(_snake_case			,      add_special_tokens=_snake_case)
      _lowerCamelCase  :				Optional[Any]      =		rust_tokenizer.encode(_snake_case			,      add_special_tokens=_snake_case)
      self.assertListEqual(_snake_case			,      _snake_case)
  
      _lowerCamelCase  :				Any      =		self.get_rust_tokenizer()
      _lowerCamelCase  :				List[Any]      =		tokenizer.encode(_snake_case)
      _lowerCamelCase  :				List[str]      =		rust_tokenizer.encode(_snake_case)
      self.assertListEqual(_snake_case			,      _snake_case)
  
      # With lower casing
      _lowerCamelCase  :				str      =		self.get_tokenizer(do_lower_case=_snake_case)
      _lowerCamelCase  :				int      =		self.get_rust_tokenizer(do_lower_case=_snake_case)
  
      _lowerCamelCase  :				Optional[Any]      =		'''UNwant\u00E9d,running'''
  
      _lowerCamelCase  :				int      =		tokenizer.tokenize(_snake_case)
      _lowerCamelCase  :				int      =		rust_tokenizer.tokenize(_snake_case)
      self.assertListEqual(_snake_case			,      _snake_case)
  
      _lowerCamelCase  :				List[Any]      =		tokenizer.encode(_snake_case			,      add_special_tokens=_snake_case)
      _lowerCamelCase  :				Optional[int]      =		rust_tokenizer.encode(_snake_case			,      add_special_tokens=_snake_case)
      self.assertListEqual(_snake_case			,      _snake_case)
  
      _lowerCamelCase  :				Optional[int]      =		self.get_rust_tokenizer()
      _lowerCamelCase  :				Dict      =		tokenizer.encode(_snake_case)
      _lowerCamelCase  :				List[Any]      =		rust_tokenizer.encode(_snake_case)
      self.assertListEqual(_snake_case			,      _snake_case)
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Dict:
      _lowerCamelCase  :				int      =		BasicTokenizer()
  
      self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")			,      ["""ah""", """\u535A""", """\u63A8""", """zz"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Union[str, Any]:
      _lowerCamelCase  :				List[str]      =		BasicTokenizer(do_lower_case=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)			,      ["""hello""", """!""", """how""", """are""", """you""", """?"""])
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,      ["""hello"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Any:
      _lowerCamelCase  :				int      =		BasicTokenizer(do_lower_case=_snake_case			,      strip_accents=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,      ["""hällo""", """!""", """how""", """are""", """you""", """?"""])
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,      ["""h\u00E9llo"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Dict:
      _lowerCamelCase  :				Optional[Any]      =		BasicTokenizer(do_lower_case=_snake_case			,      strip_accents=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,      ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,      ["""hello"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     str:
      _lowerCamelCase  :				Any      =		BasicTokenizer(do_lower_case=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,      ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,      ["""hello"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Optional[int]:
      _lowerCamelCase  :				str      =		BasicTokenizer(do_lower_case=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)			,      ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     str:
      _lowerCamelCase  :				Tuple      =		BasicTokenizer(do_lower_case=_snake_case			,      strip_accents=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,      ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Any:
      _lowerCamelCase  :				Union[str, Any]      =		BasicTokenizer(do_lower_case=_snake_case			,      strip_accents=_snake_case)
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,      ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     str:
      _lowerCamelCase  :				Any      =		BasicTokenizer(do_lower_case=_snake_case			,      never_split=["""[UNK]"""])
  
      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")			,      ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Any:
      _lowerCamelCase  :				Union[str, Any]      =		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
      _lowerCamelCase  :				Union[str, Any]      =		{}
      for i, token in enumerate(_snake_case):
       _lowerCamelCase  :				Optional[int]      =		i
      _lowerCamelCase  :				Any      =		WordpieceTokenizer(vocab=_snake_case			,      unk_token="""[UNK]""")
  
      self.assertListEqual(tokenizer.tokenize("""""")			,      [])
  
      self.assertListEqual(tokenizer.tokenize("""unwanted running""")			,      ["""un""", """##want""", """##ed""", """runn""", """##ing"""])
  
      self.assertListEqual(tokenizer.tokenize("""unwantedX running""")			,      ["""[UNK]""", """runn""", """##ing"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Optional[int]:
      self.assertTrue(_is_whitespace(""" """))
      self.assertTrue(_is_whitespace("""\t"""))
      self.assertTrue(_is_whitespace("""\r"""))
      self.assertTrue(_is_whitespace("""\n"""))
      self.assertTrue(_is_whitespace("""\u00A0"""))
  
      self.assertFalse(_is_whitespace("""A"""))
      self.assertFalse(_is_whitespace("""-"""))
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Optional[Any]:
      self.assertTrue(_is_control("""\u0005"""))
  
      self.assertFalse(_is_control("""A"""))
      self.assertFalse(_is_control(""" """))
      self.assertFalse(_is_control("""\t"""))
      self.assertFalse(_is_control("""\r"""))
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Optional[int]:
      self.assertTrue(_is_punctuation("""-"""))
      self.assertTrue(_is_punctuation("""$"""))
      self.assertTrue(_is_punctuation("""`"""))
      self.assertTrue(_is_punctuation("""."""))
  
      self.assertFalse(_is_punctuation("""A"""))
      self.assertFalse(_is_punctuation(""" """))
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Optional[Any]:
      _lowerCamelCase  :				Optional[int]      =		self.get_tokenizer()
      _lowerCamelCase  :				List[str]      =		self.get_rust_tokenizer()
  
      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(_snake_case) for t in ["""Test""", """\xad""", """test"""]]			,      [["""[UNK]"""], [], ["""[UNK]"""]])
  
      self.assertListEqual(
          [rust_tokenizer.tokenize(_snake_case) for t in ["""Test""", """\xad""", """test"""]]			,      [["""[UNK]"""], [], ["""[UNK]"""]])
  
  
  
  
  
     @slow
     def 		UpperCamelCase_     (       self)     ->     List[Any]:
      _lowerCamelCase  :				str      =		self.tokenizer_class.from_pretrained("""google/electra-base-discriminator""")
  
      _lowerCamelCase  :				List[str]      =		tokenizer.encode("""sequence builders"""			,      add_special_tokens=_snake_case)
      _lowerCamelCase  :				Dict      =		tokenizer.encode("""multi-sequence build"""			,      add_special_tokens=_snake_case)
  
      _lowerCamelCase  :				Any      =		tokenizer.build_inputs_with_special_tokens(_snake_case)
      _lowerCamelCase  :				int      =		tokenizer.build_inputs_with_special_tokens(_snake_case			,      _snake_case)
  
      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     int:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
        _lowerCamelCase  :				Any      =		self.rust_tokenizer_class.from_pretrained(_snake_case			,      **_snake_case)
  
        _lowerCamelCase  :				Tuple      =		F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
        _lowerCamelCase  :				str      =		tokenizer_r.encode_plus(
            _snake_case			,      return_attention_mask=_snake_case			,      return_token_type_ids=_snake_case			,      return_offsets_mapping=_snake_case			,      add_special_tokens=_snake_case			,      )
  
        _lowerCamelCase  :				List[Any]      =		tokenizer_r.do_lower_case if hasattr(_snake_case			,      """do_lower_case""") else False
        _lowerCamelCase  :				int      =		(
            [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''A'''),
                ((1, 2), ''','''),
                ((3, 5), '''na'''),
                ((5, 6), '''##ï'''),
                ((6, 8), '''##ve'''),
                ((9, 15), tokenizer_r.mask_token),
                ((16, 21), '''Allen'''),
                ((21, 23), '''##NL'''),
                ((23, 24), '''##P'''),
                ((25, 33), '''sentence'''),
                ((33, 34), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
            if not do_lower_case
            else [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), '''a'''),
                ((1, 2), ''','''),
                ((3, 8), '''naive'''),
                ((9, 15), tokenizer_r.mask_token),
                ((16, 21), '''allen'''),
                ((21, 23), '''##nl'''),
                ((23, 24), '''##p'''),
                ((25, 33), '''sentence'''),
                ((33, 34), '''.'''),
                ((0, 0), tokenizer_r.sep_token),
            ]
        )
  
        self.assertEqual(
            [e[1] for e in expected_results]			,      tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
        self.assertEqual([e[0] for e in expected_results]			,      tokens["""offset_mapping"""])
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Union[str, Any]:
      _lowerCamelCase  :				List[Any]      =		['''的''', '''人''', '''有''']
      _lowerCamelCase  :				Dict      =		''''''.join(_snake_case)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
        _lowerCamelCase  :				str      =		True
        _lowerCamelCase  :				str      =		self.tokenizer_class.from_pretrained(_snake_case			,      **_snake_case)
        _lowerCamelCase  :				List[str]      =		self.rust_tokenizer_class.from_pretrained(_snake_case			,      **_snake_case)
  
        _lowerCamelCase  :				Union[str, Any]      =		tokenizer_p.encode(_snake_case			,      add_special_tokens=_snake_case)
        _lowerCamelCase  :				Dict      =		tokenizer_r.encode(_snake_case			,      add_special_tokens=_snake_case)
  
        _lowerCamelCase  :				Optional[int]      =		tokenizer_r.convert_ids_to_tokens(_snake_case)
        _lowerCamelCase  :				Union[str, Any]      =		tokenizer_p.convert_ids_to_tokens(_snake_case)
  
        # it is expected that each Chinese character is not preceded by "##"
        self.assertListEqual(_snake_case			,      _snake_case)
        self.assertListEqual(_snake_case			,      _snake_case)
  
        _lowerCamelCase  :				Union[str, Any]      =		False
        _lowerCamelCase  :				Any      =		self.rust_tokenizer_class.from_pretrained(_snake_case			,      **_snake_case)
        _lowerCamelCase  :				Dict      =		self.tokenizer_class.from_pretrained(_snake_case			,      **_snake_case)
  
        _lowerCamelCase  :				int      =		tokenizer_r.encode(_snake_case			,      add_special_tokens=_snake_case)
        _lowerCamelCase  :				Dict      =		tokenizer_p.encode(_snake_case			,      add_special_tokens=_snake_case)
  
        _lowerCamelCase  :				List[Any]      =		tokenizer_r.convert_ids_to_tokens(_snake_case)
        _lowerCamelCase  :				Tuple      =		tokenizer_p.convert_ids_to_tokens(_snake_case)
  
        # it is expected that only the first Chinese character is not preceded by "##".
        _lowerCamelCase  :				Any      =		[
            F'##{token}' if idx != 0 else token for idx, token in enumerate(_snake_case)
        ]
        self.assertListEqual(_snake_case			,      _snake_case)
        self.assertListEqual(_snake_case			,      _snake_case)
  
  
style_context: 
  
  
  """simple docstring"""
  from typing import List, Optional, Tuple
  
  from tokenizers import pre_tokenizers
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_clip import CLIPTokenizer
  
  
  UpperCAmelCase   =	logging.get_logger(__name__)
  
  UpperCAmelCase   =	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}
  
  UpperCAmelCase   =	{
      """vocab_file""": {
          """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
      },
      """merges_file""": {
          """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
      },
      """tokenizer_file""": {
          """openai/clip-vit-base-patch32""": (
              """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
          ),
      },
  }
  
  UpperCAmelCase   =	{
      """openai/clip-vit-base-patch32""": 77,
  }
  
  
  
  
  class   lowercase__							(				A_							):
     __UpperCAmelCase       					=				VOCAB_FILES_NAMES
     __UpperCAmelCase       					=				PRETRAINED_VOCAB_FILES_MAP
     __UpperCAmelCase       					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCAmelCase       					=				['''input_ids''', '''attention_mask''']
     __UpperCAmelCase       					=				CLIPTokenizer
  
  
  
  
  
     def __init__(       self			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE="<|endoftext|>"			,      SCREAMING_SNAKE_CASE="<|startoftext|>"			,      SCREAMING_SNAKE_CASE="<|endoftext|>"			,      SCREAMING_SNAKE_CASE="<|endoftext|>"			,      **SCREAMING_SNAKE_CASE			,      )     ->     List[Any]:
        super().__init__(
            SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      tokenizer_file=SCREAMING_SNAKE_CASE			,      unk_token=SCREAMING_SNAKE_CASE			,      bos_token=SCREAMING_SNAKE_CASE			,      eos_token=SCREAMING_SNAKE_CASE			,      pad_token=SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE			,      )
  
        if not isinstance(self.backend_tokenizer.pre_tokenizer			,      pre_tokenizers.Sequence):
           raise ValueError(
               """The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"""
               """ heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"""
               """ to be compatible with this version.The easiest way to do so is"""
               """ `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"""
               """ to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"""
               """ transformers.""")
  
        self._wrap_decode_method_backend_tokenizer()
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Union[str, Any]:
        _lowerCamelCase  :				str      =		self.backend_tokenizer.decode
  
        def new_decode_method(*SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE):
           _lowerCamelCase  :				Any      =		orig_decode_method(*SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
           _lowerCamelCase  :				Optional[int]      =		text.replace(self.backend_tokenizer.model.end_of_word_suffix			,      """ """).strip()
           return text
  
        _lowerCamelCase  :				List[str]      =		new_decode_method
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None)     ->     List[int]:
        _lowerCamelCase  :				str      =		[self.bos_token_id]
        _lowerCamelCase  :				Union[str, Any]      =		[self.eos_token_id]
  
        if token_ids_a is None:
           return bos_token + token_ids_a + eos_token
        return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None)     ->     List[int]:
        _lowerCamelCase  :				Any      =		[self.bos_token_id]
        _lowerCamelCase  :				int      =		[self.eos_token_id]
  
        if token_ids_a is None:
           return len(bos_token + token_ids_a + eos_token) * [0]
        return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token) * [0]
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None)     ->     Tuple[str]:
        _lowerCamelCase  :				Dict      =		self._tokenizer.model.save(SCREAMING_SNAKE_CASE			,      name=SCREAMING_SNAKE_CASE)
        return tuple(SCREAMING_SNAKE_CASE)
  
  