code: 
  
  
  
  import json
  import os
  import unittest
  
  from transformers import HerbertTokenizer, HerbertTokenizerFast
  from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
  from transformers.testing_utils import get_tests_dir, require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  @require_tokenizers
  class        lowerCAmelCase_	(						a__ ,					unittest.TestCase							):
         UpperCAmelCase__       :      List[Any]       =						HerbertTokenizer
         UpperCAmelCase__       :      List[str]       =						HerbertTokenizerFast
         UpperCAmelCase__       :      List[str]       =						True
         def       snake_case_ (    self )  ->	Optional[int]:
            super().setUp()
  
            # Use a simpler test file without japanese/chinese characters
            with open(F"""{get_tests_dir()}/fixtures/sample_text_no_unicode.txt""",    encoding='utf-8' ) as f_data:
               UpperCamelCase					:						Union[str, Any]					=			f_data.read().replace('\n\n',    '\n' ).strip()
  
            UpperCamelCase					:						Union[str, Any]					=			[
                '<s>',
                '</s>',
                'l',
                'o',
                'w',
                'e',
                'r',
                's',
                't',
                'i',
                'd',
                'n',
                'w</w>',
                'r</w>',
                't</w>',
                'lo',
                'low',
                'er</w>',
                'low</w>',
                'lowest</w>',
                'newer</w>',
                'wider</w>',
                ',</w>',
                '<unk>',
            ]
            UpperCamelCase					:						Any					=			dict(zip(SCREAMING_SNAKE_CASE_,    range(len(SCREAMING_SNAKE_CASE_ ) ) ) )
            UpperCamelCase					:						Optional[int]					=			['l o 123', 'lo w 1456', 'e r</w> 1789', '']
  
            UpperCamelCase					:						str					=			os.path.join(self.tmpdirname,    VOCAB_FILES_NAMES['vocab_file'] )
            UpperCamelCase					:						str					=			os.path.join(self.tmpdirname,    VOCAB_FILES_NAMES['merges_file'] )
            with open(self.vocab_file,    'w' ) as fp:
               fp.write(json.dumps(SCREAMING_SNAKE_CASE_ ) )
            with open(self.merges_file,    'w' ) as fp:
               fp.write('\n'.join(SCREAMING_SNAKE_CASE_ ) )
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_ )  ->	Dict:
            UpperCamelCase					:						Any					=			'lower newer'
            UpperCamelCase					:						Tuple					=			'lower newer'
            return input_text, output_text
         def       snake_case_ (    self )  ->	Optional[int]:
            UpperCamelCase					:						Union[str, Any]					=			self.tokenizer_class(vocab_file=self.vocab_file,    merges_file=self.merges_file )
  
            UpperCamelCase					:						Optional[int]					=			'lower'
            UpperCamelCase					:						Any					=			['low', 'er</w>']
            UpperCamelCase					:						Any					=			tokenizer.tokenize(SCREAMING_SNAKE_CASE_ )
            self.assertListEqual(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
  
            UpperCamelCase					:						Dict					=			tokens + ['<unk>']
            UpperCamelCase					:						Optional[Any]					=			[16, 17, 23]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ ),    SCREAMING_SNAKE_CASE_ )
         def       snake_case_ (    self )  ->	int:
            if not self.test_rust_tokenizer:
               return
  
            UpperCamelCase					:						Union[str, Any]					=			self.get_tokenizer()
            UpperCamelCase					:						str					=			self.get_rust_tokenizer()
  
            UpperCamelCase					:						Tuple					=			'lower,newer'
  
            UpperCamelCase					:						Any					=			tokenizer.tokenize(SCREAMING_SNAKE_CASE_ )
            UpperCamelCase					:						int					=			rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_ )
            self.assertListEqual(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
  
            UpperCamelCase					:						Dict					=			tokenizer.encode(SCREAMING_SNAKE_CASE_,    add_special_tokens=SCREAMING_SNAKE_CASE_ )
            UpperCamelCase					:						Optional[int]					=			rust_tokenizer.encode(SCREAMING_SNAKE_CASE_,    add_special_tokens=SCREAMING_SNAKE_CASE_ )
            self.assertListEqual(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
  
            UpperCamelCase					:						Tuple					=			self.get_rust_tokenizer()
            UpperCamelCase					:						Union[str, Any]					=			tokenizer.encode(SCREAMING_SNAKE_CASE_ )
            UpperCamelCase					:						List[str]					=			rust_tokenizer.encode(SCREAMING_SNAKE_CASE_ )
            self.assertListEqual(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
  
  
  
  
  
  
         @slow
         def       snake_case_ (    self )  ->	str:
            UpperCamelCase					:						Optional[Any]					=			self.tokenizer_class.from_pretrained('allegro/herbert-base-cased' )
  
            UpperCamelCase					:						Dict					=			tokenizer.encode('konstruowanie sekwencji',    add_special_tokens=SCREAMING_SNAKE_CASE_ )
            UpperCamelCase					:						Union[str, Any]					=			tokenizer.encode('konstruowanie wielu sekwencji',    add_special_tokens=SCREAMING_SNAKE_CASE_ )
  
            UpperCamelCase					:						Optional[Any]					=			tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_ )
            UpperCamelCase					:						List[Any]					=			tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
  
            assert encoded_sentence == [0] + text + [2]
            assert encoded_pair == [0] + text + [2] + text_a + [2]
  
  
  
  
style_context: 
  
  
  
  import unittest
  
  import numpy as np
  
  from transformers.testing_utils import require_torch, require_vision
  from transformers.utils import is_torch_available, is_vision_available
  
  from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs
  
  
  if is_torch_available():
      import torch
  
  if is_vision_available():
      from PIL import Image
  
      from transformers import ViltImageProcessor
  
  
  
  class        lowerCAmelCase_	(						unittest.TestCase							):
         def __init__(    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=7,    SCREAMING_SNAKE_CASE_=3,    SCREAMING_SNAKE_CASE_=18,    SCREAMING_SNAKE_CASE_=30,    SCREAMING_SNAKE_CASE_=400,    SCREAMING_SNAKE_CASE_=True,    SCREAMING_SNAKE_CASE_=None,    SCREAMING_SNAKE_CASE_=2,    SCREAMING_SNAKE_CASE_=True,    SCREAMING_SNAKE_CASE_=[0.5, 0.5, 0.5],    SCREAMING_SNAKE_CASE_=[0.5, 0.5, 0.5],    )  ->	List[Any]:
            UpperCamelCase					:						Optional[int]					=			size if size is not None else {'shortest_edge': 30}
            UpperCamelCase					:						Dict					=			parent
            UpperCamelCase					:						str					=			batch_size
            UpperCamelCase					:						str					=			num_channels
            UpperCamelCase					:						Optional[int]					=			image_size
            UpperCamelCase					:						List[Any]					=			min_resolution
            UpperCamelCase					:						Tuple					=			max_resolution
            UpperCamelCase					:						Tuple					=			do_resize
            UpperCamelCase					:						Any					=			size
            UpperCamelCase					:						int					=			size_divisor
            UpperCamelCase					:						str					=			do_normalize
            UpperCamelCase					:						Optional[int]					=			image_mean
            UpperCamelCase					:						List[str]					=			image_std
         def       snake_case_ (    self )  ->	List[Any]:
            return {
                "image_mean": self.image_mean,
                "image_std": self.image_std,
                "do_normalize": self.do_normalize,
                "do_resize": self.do_resize,
                "size": self.size,
                "size_divisor": self.size_divisor,
            }
  
  
  
  
  
  
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=False )  ->	Union[str, Any]:
            if not batched:
               UpperCamelCase					:						Optional[Any]					=			self.size['shortest_edge']
               UpperCamelCase					:						int					=			image_inputs[0]
               if isinstance(SCREAMING_SNAKE_CASE_,    Image.Image ):
                  UpperCamelCase			,  UpperCamelCase					:						List[Any]					=			image.size
               else:
                  UpperCamelCase			,  UpperCamelCase					:						List[Any]					=			image.shape[1], image.shape[2]
               UpperCamelCase					:						Any					=			size / min(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
               if h < w:
                  UpperCamelCase			,  UpperCamelCase					:						Optional[int]					=			size, scale * w
               else:
                  UpperCamelCase			,  UpperCamelCase					:						List[str]					=			scale * h, size
  
               UpperCamelCase					:						int					=			int((1333 / 800) * size )
               if max(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ ) > max_size:
                  UpperCamelCase					:						Any					=			max_size / max(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
                  UpperCamelCase					:						List[Any]					=			newh * scale
                  UpperCamelCase					:						Optional[int]					=			neww * scale
  
               UpperCamelCase			,  UpperCamelCase					:						List[Any]					=			int(newh + 0.5 ), int(neww + 0.5 )
               UpperCamelCase			,  UpperCamelCase					:						int					=			(
                   newh // self.size_divisor * self.size_divisor,
                   neww // self.size_divisor * self.size_divisor,
               )
  
            else:
               UpperCamelCase					:						Dict					=			[]
               for image in image_inputs:
                  UpperCamelCase			,  UpperCamelCase					:						Union[str, Any]					=			self.get_expected_values([image] )
                  expected_values.append((expected_height, expected_width) )
               UpperCamelCase					:						List[str]					=			max(SCREAMING_SNAKE_CASE_,    key=lambda SCREAMING_SNAKE_CASE_ : item[0] )[0]
               UpperCamelCase					:						List[str]					=			max(SCREAMING_SNAKE_CASE_,    key=lambda SCREAMING_SNAKE_CASE_ : item[1] )[1]
  
            return expected_height, expected_width
  
  
  
  
  
  
  @require_torch
  @require_vision
  class        lowerCAmelCase_	(						a__ ,					unittest.TestCase							):
         UpperCAmelCase__       :      List[Any]       =						ViltImageProcessor if is_vision_available() else None
         def       snake_case_ (    self )  ->	Union[str, Any]:
            UpperCamelCase					:						Dict					=			ViltImageProcessingTester(self )
         @property
         def       snake_case_ (    self )  ->	Dict:
            return self.image_processor_tester.prepare_image_processor_dict()
         def       snake_case_ (    self )  ->	Union[str, Any]:
            UpperCamelCase					:						List[str]					=			self.image_processing_class(**self.image_processor_dict )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'image_mean' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'image_std' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'do_normalize' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'do_resize' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'size' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'size_divisor' ) )
         def       snake_case_ (    self )  ->	List[Any]:
            UpperCamelCase					:						Union[str, Any]					=			self.image_processing_class.from_dict(self.image_processor_dict )
            self.assertEqual(image_processor.size,    {'shortest_edge': 30} )
  
            UpperCamelCase					:						Union[str, Any]					=			self.image_processing_class.from_dict(self.image_processor_dict,    size=42 )
            self.assertEqual(image_processor.size,    {'shortest_edge': 42} )
         def       snake_case_ (    self )  ->	Dict:
            pass
         def       snake_case_ (    self )  ->	str:
            # Initialize image_processing
            UpperCamelCase					:						int					=			self.image_processing_class(**self.image_processor_dict )
            # create random PIL images
            UpperCamelCase					:						Any					=			prepare_image_inputs(self.image_processor_tester,    equal_resolution=SCREAMING_SNAKE_CASE_ )
            for image in image_inputs:
               self.assertIsInstance(SCREAMING_SNAKE_CASE_,    Image.Image )
  
            # Test not batched input
            UpperCamelCase					:						int					=			image_processing(image_inputs[0],    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Dict					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (1, self.image_processor_tester.num_channels, expected_height, expected_width),    )
  
            # Test batched
            UpperCamelCase					:						str					=			image_processing(SCREAMING_SNAKE_CASE_,    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Tuple					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_,    batched=SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    expected_height,
                    expected_width,
                ),    )
         def       snake_case_ (    self )  ->	List[str]:
            # Initialize image_processing
            UpperCamelCase					:						Optional[int]					=			self.image_processing_class(**self.image_processor_dict )
            # create random numpy tensors
            UpperCamelCase					:						Optional[Any]					=			prepare_image_inputs(self.image_processor_tester,    equal_resolution=SCREAMING_SNAKE_CASE_,    numpify=SCREAMING_SNAKE_CASE_ )
            for image in image_inputs:
               self.assertIsInstance(SCREAMING_SNAKE_CASE_,    np.ndarray )
  
            # Test not batched input
            UpperCamelCase					:						Optional[Any]					=			image_processing(image_inputs[0],    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						str					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (1, self.image_processor_tester.num_channels, expected_height, expected_width),    )
  
            # Test batched
            UpperCamelCase					:						Dict					=			image_processing(SCREAMING_SNAKE_CASE_,    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Optional[Any]					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_,    batched=SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    expected_height,
                    expected_width,
                ),    )
  
  
  
  
  
  
         def       snake_case_ (    self )  ->	Optional[Any]:
            # Initialize image_processing
            UpperCamelCase					:						Optional[int]					=			self.image_processing_class(**self.image_processor_dict )
            # create random PyTorch tensors
            UpperCamelCase					:						Dict					=			prepare_image_inputs(self.image_processor_tester,    equal_resolution=SCREAMING_SNAKE_CASE_,    torchify=SCREAMING_SNAKE_CASE_ )
            for image in image_inputs:
               self.assertIsInstance(SCREAMING_SNAKE_CASE_,    torch.Tensor )
  
            # Test not batched input
            UpperCamelCase					:						List[str]					=			image_processing(image_inputs[0],    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Union[str, Any]					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (1, self.image_processor_tester.num_channels, expected_height, expected_width),    )
  
            # Test batched
            UpperCamelCase					:						List[Any]					=			image_processing(SCREAMING_SNAKE_CASE_,    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Dict					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_,    batched=SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    expected_height,
                    expected_width,
                ),    )
  
  
  
  