






'''simple docstring'''







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class      __SCREAMING_SNAKE_CASE					(__A       ,					unittest.TestCase					):


  """simple docstring"""


  _a     :   Union[str, Any]			 =    HerbertTokenizer
  _a     :   Optional[int]			 =    HerbertTokenizerFast
  _a     :   Optional[int]			 =    True







  def       _a					( self  ):






         """simple docstring"""


         super().setUp()

         # Use a simpler test file without japanese/chinese characters
         with open(f'{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'      ,  encoding='utf-8'  ) as f_data:
                a_	    = f_data.read().replace('\n\n'      ,  '\n'  ).strip()

         a_	    = [
             '<s>',
             '</s>',
             'l',
             'o',
             'w',
             'e',
             'r',
             's',
             't',
             'i',
             'd',
             'n',
             'w</w>',
             'r</w>',
             't</w>',
             'lo',
             'low',
             'er</w>',
             'low</w>',
             'lowest</w>',
             'newer</w>',
             'wider</w>',
             ',</w>',
             '<unk>',
         ]
         a_	    = dict(zip(UpperCamelCase__      ,  range(len(UpperCamelCase__  )  )  )  )
         a_	    = ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

         a_	    = os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['vocab_file']  )
         a_	    = os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['merges_file']  )
         with open(self.vocab_file      ,  'w'  ) as fp:
                fp.write(json.dumps(UpperCamelCase__  )  )
         with open(self.merges_file      ,  'w'  ) as fp:
                fp.write('\n'.join(UpperCamelCase__  )  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = 'lower newer'
         a_	    = 'lower newer'
         return input_text, output_text







  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.tokenizer_class(vocab_file=self.vocab_file      ,  merges_file=self.merges_file  )

         a_	    = 'lower'
         a_	    = ['low', 'er</w>']
         a_	    = tokenizer.tokenize(UpperCamelCase__  )
         self.assertListEqual(UpperCamelCase__      ,  UpperCamelCase__  )

         a_	    = tokens + ['<unk>']
         a_	    = [16, 17, 23]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  )      ,  UpperCamelCase__  )







  def       _a					( self  ):






         """simple docstring"""


         if not self.test_rust_tokenizer:
                return

         a_	    = self.get_tokenizer()
         a_	    = self.get_rust_tokenizer()

         a_	    = 'lower,newer'

         a_	    = tokenizer.tokenize(UpperCamelCase__  )
         a_	    = rust_tokenizer.tokenize(UpperCamelCase__  )
         self.assertListEqual(UpperCamelCase__      ,  UpperCamelCase__  )

         a_	    = tokenizer.encode(UpperCamelCase__      ,  add_special_tokens=UpperCamelCase__  )
         a_	    = rust_tokenizer.encode(UpperCamelCase__      ,  add_special_tokens=UpperCamelCase__  )
         self.assertListEqual(UpperCamelCase__      ,  UpperCamelCase__  )

         a_	    = self.get_rust_tokenizer()
         a_	    = tokenizer.encode(UpperCamelCase__  )
         a_	    = rust_tokenizer.encode(UpperCamelCase__  )
         self.assertListEqual(UpperCamelCase__      ,  UpperCamelCase__  )







  @slow
  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.tokenizer_class.from_pretrained('allegro/herbert-base-cased'  )

         a_	    = tokenizer.encode('konstruowanie sekwencji'      ,  add_special_tokens=UpperCamelCase__  )
         a_	    = tokenizer.encode('konstruowanie wielu sekwencji'      ,  add_special_tokens=UpperCamelCase__  )

         a_	    = tokenizer.build_inputs_with_special_tokens(UpperCamelCase__  )
         a_	    = tokenizer.build_inputs_with_special_tokens(UpperCamelCase__      ,  UpperCamelCase__  )

         assert encoded_sentence == [0] + text + [2]
         assert encoded_pair == [0] + text + [2] + text_a + [2]












'''simple docstring'''







import re


def 			__UpperCamelCase				(			lowercase_					:     int      ):
       """simple docstring"""




       def docstring_decorator(lowercase_					:     int      ):
              a_	    = fn.__doc__
              a_	    = func_doc.split('\n'      )
              a_	    = 0
              while i < len(lowercase_      ) and re.search(R'^\s*Examples?:\s*$'      ,					lines[i]      ) is None:
                     i += 1
              if i < len(lowercase_      ):
                     a_	    = example_docstring
                     a_	    = '\n'.join(lowercase_      )
              else:
                     raise ValueError(
                         F'The function {fn} should have an empty \'Examples:\' in its docstring as placeholder, '
                         F'current docstring is:\n{func_doc}'      )
              a_	    = func_doc
              return fn

       return docstring_decorator




