code: import json
  import os
  import unittest
  
  from transformers import HerbertTokenizer, HerbertTokenizerFast
  from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
  from transformers.testing_utils import get_tests_dir, require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  @require_tokenizers
  class _lowercase					(       _UpperCAmelCase   ,				unittest.TestCase						):
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  							lowerCAmelCase__       =					HerbertTokenizer
  							lowerCAmelCase__       =					HerbertTokenizerFast
  							lowerCAmelCase__       =					True
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														super().setUp()
  
  														# Use a simpler test file without japanese/chinese characters
  														with open(F'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''     ,   encoding="""utf-8"""  ) as f_data:
  																					_lowercase				=	f_data.read().replace("""\n\n"""     ,   """\n"""  ).strip()
  
  														_lowercase				=	[
  														    """<s>""",
  														    """</s>""",
  														    """l""",
  														    """o""",
  														    """w""",
  														    """e""",
  														    """r""",
  														    """s""",
  														    """t""",
  														    """i""",
  														    """d""",
  														    """n""",
  														    """w</w>""",
  														    """r</w>""",
  														    """t</w>""",
  														    """lo""",
  														    """low""",
  														    """er</w>""",
  														    """low</w>""",
  														    """lowest</w>""",
  														    """newer</w>""",
  														    """wider</w>""",
  														    """,</w>""",
  														    """<unk>""",
  														]
  														_lowercase				=	dict(zip(UpperCAmelCase     ,   range(len(UpperCAmelCase  )  )  )  )
  														_lowercase				=	["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]
  
  														_lowercase				=	os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["""vocab_file"""]  )
  														_lowercase				=	os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["""merges_file"""]  )
  														with open(self.vocab_file     ,   """w"""  ) as fp:
  																					fp.write(json.dumps(UpperCAmelCase  )  )
  														with open(self.merges_file     ,   """w"""  ) as fp:
  																					fp.write("""\n""".join(UpperCAmelCase  )  )
  
  
  
  							def     _UpperCAmelCase       (    self     ,   UpperCAmelCase  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	"""lower newer"""
  														_lowercase				=	"""lower newer"""
  														return input_text, output_text
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.tokenizer_class(vocab_file=self.vocab_file     ,   merges_file=self.merges_file  )
  
  														_lowercase				=	"""lower"""
  														_lowercase				=	["""low""", """er</w>"""]
  														_lowercase				=	tokenizer.tokenize(UpperCAmelCase  )
  														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )
  
  														_lowercase				=	tokens + ["""<unk>"""]
  														_lowercase				=	[16, 17, 23]
  														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase  )     ,   UpperCAmelCase  )
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														if not self.test_rust_tokenizer:
  																					return
  
  														_lowercase				=	self.get_tokenizer()
  														_lowercase				=	self.get_rust_tokenizer()
  
  														_lowercase				=	"""lower,newer"""
  
  														_lowercase				=	tokenizer.tokenize(UpperCAmelCase  )
  														_lowercase				=	rust_tokenizer.tokenize(UpperCAmelCase  )
  														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )
  
  														_lowercase				=	tokenizer.encode(UpperCAmelCase     ,   add_special_tokens=UpperCAmelCase  )
  														_lowercase				=	rust_tokenizer.encode(UpperCAmelCase     ,   add_special_tokens=UpperCAmelCase  )
  														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )
  
  														_lowercase				=	self.get_rust_tokenizer()
  														_lowercase				=	tokenizer.encode(UpperCAmelCase  )
  														_lowercase				=	rust_tokenizer.encode(UpperCAmelCase  )
  														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )
  
  
  
  							@slow
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.tokenizer_class.from_pretrained("""allegro/herbert-base-cased"""  )
  
  														_lowercase				=	tokenizer.encode("""konstruowanie sekwencji"""     ,   add_special_tokens=UpperCAmelCase  )
  														_lowercase				=	tokenizer.encode("""konstruowanie wielu sekwencji"""     ,   add_special_tokens=UpperCAmelCase  )
  
  														_lowercase				=	tokenizer.build_inputs_with_special_tokens(UpperCAmelCase  )
  														_lowercase				=	tokenizer.build_inputs_with_special_tokens(UpperCAmelCase     ,   UpperCAmelCase  )
  
  														assert encoded_sentence == [0] + text + [2]
  														assert encoded_pair == [0] + text + [2] + text_a + [2]
  
  
  
style_context: def 					__lowerCAmelCase			(  _A					):
  
  
  
  							"""simple docstring"""
  
  
  							if len(_A					) <= 1:
  														return data
  							else:
  														return [
  														    *quick_sort([e for e in data[1:] if e <= data[0]]					),
  														    data[0],
  														    *quick_sort([e for e in data[1:] if e > data[0]]					),
  														]
  
  
  if __name__ == "__main__":
  				import doctest
  
  				doctest.testmod()
  
  
  