code: 
  
  
  
  
  
  
  import json
  import os
  import unittest
  
  from transformers import HerbertTokenizer, HerbertTokenizerFast
  from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
  from transformers.testing_utils import get_tests_dir, require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class     __a						(					A__ ,     unittest.TestCase ):
   _lowerCAmelCase : Optional[int] 				=				HerbertTokenizer
   _lowerCAmelCase : str 				=				HerbertTokenizerFast
   _lowerCAmelCase : Tuple 				=				True
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        super().setUp()
  
        # Use a simpler test file without japanese/chinese characters
        with open(F'{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'							,     encoding="utf-8"		) as f_data:
             UpperCamelCase__    :     Optional[int]        =						f_data.read().replace("\n\n"							,     "\n"		).strip()
  
        UpperCamelCase__    :     Dict        =						[
            "<s>",
            "</s>",
            "l",
            "o",
            "w",
            "e",
            "r",
            "s",
            "t",
            "i",
            "d",
            "n",
            "w</w>",
            "r</w>",
            "t</w>",
            "lo",
            "low",
            "er</w>",
            "low</w>",
            "lowest</w>",
            "newer</w>",
            "wider</w>",
            ",</w>",
            "<unk>",
        ]
        UpperCamelCase__    :     Optional[Any]        =						dict(zip(SCREAMING_SNAKE_CASE							,     range(len(SCREAMING_SNAKE_CASE		)		)		)		)
        UpperCamelCase__    :     Union[str, Any]        =						["l o 123", "lo w 1456", "e r</w> 1789", ""]
  
        UpperCamelCase__    :     List[Any]        =						os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["vocab_file"]		)
        UpperCamelCase__    :     Union[str, Any]        =						os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["merges_file"]		)
        with open(self.vocab_file							,     "w"		) as fp:
             fp.write(json.dumps(SCREAMING_SNAKE_CASE		)		)
        with open(self.merges_file							,     "w"		) as fp:
             fp.write("\n".join(SCREAMING_SNAKE_CASE		)		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]							,     SCREAMING_SNAKE_CASE      :     Optional[int]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Dict        =						"lower newer"
        UpperCamelCase__    :     Union[str, Any]        =						"lower newer"
        return input_text, output_text
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     str		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[Any]        =						self.tokenizer_class(vocab_file=self.vocab_file							,     merges_file=self.merges_file		)
  
        UpperCamelCase__    :     List[Any]        =						"lower"
        UpperCamelCase__    :     Dict        =						["low", "er</w>"]
        UpperCamelCase__    :     Optional[Any]        =						tokenizer.tokenize(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     Dict        =						tokens + ["<unk>"]
        UpperCamelCase__    :     List[Any]        =						[16, 17, 23]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)							,     SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        if not self.test_rust_tokenizer:
             return
  
        UpperCamelCase__    :     int        =						self.get_tokenizer()
        UpperCamelCase__    :     Union[str, Any]        =						self.get_rust_tokenizer()
  
        UpperCamelCase__    :     Optional[int]        =						"lower,newer"
  
        UpperCamelCase__    :     List[str]        =						tokenizer.tokenize(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Union[str, Any]        =						rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     int        =						tokenizer.encode(SCREAMING_SNAKE_CASE							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Optional[Any]        =						rust_tokenizer.encode(SCREAMING_SNAKE_CASE							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     List[Any]        =						self.get_rust_tokenizer()
        UpperCamelCase__    :     Tuple        =						tokenizer.encode(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     List[str]        =						rust_tokenizer.encode(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   @slow
   def       __lowercase      (	self      :     Any		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Dict        =						self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"		)
  
        UpperCamelCase__    :     Union[str, Any]        =						tokenizer.encode("konstruowanie sekwencji"							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Tuple        =						tokenizer.encode("konstruowanie wielu sekwencji"							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     str        =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Dict        =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        assert encoded_sentence == [0] + text + [2]
        assert encoded_pair == [0] + text + [2] + text_a + [2]
style_context: 
  
  
  
  
  
  
  from __future__ import annotations
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      __lowerCAmelCase    )				->			list[int]:
       assert (
           isinstance(__lowerCAmelCase      ,					__lowerCAmelCase    ) and mask > 0
       ), f'mask needs to be positive integer, your input {mask}'
       UpperCamelCase__    :     Any        =						[]
       UpperCamelCase__    :     int        =						mask
  
       while submask:
            all_submasks.append(__lowerCAmelCase    )
            UpperCamelCase__    :     List[str]        =						(submask - 1) & mask
  
       return all_submasks
  
  
  if __name__ == "__main__":
   import doctest
  
   doctest.testmod()