code: 
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import pre_tokenizers, processors
  
  from ...tokenization_utils_base import AddedToken, BatchEncoding
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_longformer import LongformerTokenizer
  
  
  __UpperCAmelCase      = logging.get_logger(__name__)
  
  __UpperCAmelCase      = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}
  
  __UpperCAmelCase      = {
      '''vocab_file''': {
          '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
          '''allenai/longformer-large-4096''': (
              '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
          ),
          '''allenai/longformer-large-4096-finetuned-triviaqa''': (
              '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
          ),
          '''allenai/longformer-base-4096-extra.pos.embd.only''': (
              '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
          ),
          '''allenai/longformer-large-4096-extra.pos.embd.only''': (
              '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
          ),
      },
      '''merges_file''': {
          '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
          '''allenai/longformer-large-4096''': (
              '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
          ),
          '''allenai/longformer-large-4096-finetuned-triviaqa''': (
              '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
          ),
          '''allenai/longformer-base-4096-extra.pos.embd.only''': (
              '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
          ),
          '''allenai/longformer-large-4096-extra.pos.embd.only''': (
              '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
          ),
      },
      '''tokenizer_file''': {
          '''allenai/longformer-base-4096''': (
              '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'''
          ),
          '''allenai/longformer-large-4096''': (
              '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'''
          ),
          '''allenai/longformer-large-4096-finetuned-triviaqa''': (
              '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'''
          ),
          '''allenai/longformer-base-4096-extra.pos.embd.only''': (
              '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
          ),
          '''allenai/longformer-large-4096-extra.pos.embd.only''': (
              '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
          ),
      },
  }
  
  __UpperCAmelCase      = {
      '''allenai/longformer-base-4096''': 4_096,
      '''allenai/longformer-large-4096''': 4_096,
      '''allenai/longformer-large-4096-finetuned-triviaqa''': 4_096,
      '''allenai/longformer-base-4096-extra.pos.embd.only''': 4_096,
      '''allenai/longformer-large-4096-extra.pos.embd.only''': 4_096,
  }
  
  
  
  class        lowerCAmelCase_	(						a__							):
         UpperCAmelCase__       :      Optional[int]       =						VOCAB_FILES_NAMES
         UpperCAmelCase__       :      str       =						PRETRAINED_VOCAB_FILES_MAP
         UpperCAmelCase__       :      Dict       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
         UpperCAmelCase__       :      List[str]       =						["input_ids", "attention_mask"]
         UpperCAmelCase__       :      Any       =						LongformerTokenizer
         def __init__(    self,    SCREAMING_SNAKE_CASE_=None,    SCREAMING_SNAKE_CASE_=None,    SCREAMING_SNAKE_CASE_=None,    SCREAMING_SNAKE_CASE_="replace",    SCREAMING_SNAKE_CASE_="<s>",    SCREAMING_SNAKE_CASE_="</s>",    SCREAMING_SNAKE_CASE_="</s>",    SCREAMING_SNAKE_CASE_="<s>",    SCREAMING_SNAKE_CASE_="<unk>",    SCREAMING_SNAKE_CASE_="<pad>",    SCREAMING_SNAKE_CASE_="<mask>",    SCREAMING_SNAKE_CASE_=False,    SCREAMING_SNAKE_CASE_=True,    **SCREAMING_SNAKE_CASE_,    )  ->	Dict:
            super().__init__(
                SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_,    tokenizer_file=SCREAMING_SNAKE_CASE_,    errors=SCREAMING_SNAKE_CASE_,    bos_token=SCREAMING_SNAKE_CASE_,    eos_token=SCREAMING_SNAKE_CASE_,    sep_token=SCREAMING_SNAKE_CASE_,    cls_token=SCREAMING_SNAKE_CASE_,    unk_token=SCREAMING_SNAKE_CASE_,    pad_token=SCREAMING_SNAKE_CASE_,    mask_token=SCREAMING_SNAKE_CASE_,    add_prefix_space=SCREAMING_SNAKE_CASE_,    trim_offsets=SCREAMING_SNAKE_CASE_,    **SCREAMING_SNAKE_CASE_,    )
  
            UpperCamelCase					:						List[str]					=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
            if pre_tok_state.get('add_prefix_space',    SCREAMING_SNAKE_CASE_ ) != add_prefix_space:
               UpperCamelCase					:						List[Any]					=			getattr(SCREAMING_SNAKE_CASE_,    pre_tok_state.pop('type' ) )
               UpperCamelCase					:						str					=			add_prefix_space
               UpperCamelCase					:						Tuple					=			pre_tok_class(**SCREAMING_SNAKE_CASE_ )
  
            UpperCamelCase					:						str					=			add_prefix_space
  
            UpperCamelCase					:						str					=			'post_processor'
            UpperCamelCase					:						Dict					=			getattr(self.backend_tokenizer,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
            if tokenizer_component_instance:
               UpperCamelCase					:						Union[str, Any]					=			json.loads(tokenizer_component_instance.__getstate__() )
  
               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                  UpperCamelCase					:						List[str]					=			tuple(state['sep'] )
               if "cls" in state:
                  UpperCamelCase					:						Optional[int]					=			tuple(state['cls'] )
  
               UpperCamelCase					:						int					=			False
  
               if state.get('add_prefix_space',    SCREAMING_SNAKE_CASE_ ) != add_prefix_space:
                  UpperCamelCase					:						Optional[int]					=			add_prefix_space
                  UpperCamelCase					:						Union[str, Any]					=			True
  
               if state.get('trim_offsets',    SCREAMING_SNAKE_CASE_ ) != trim_offsets:
                  UpperCamelCase					:						int					=			trim_offsets
                  UpperCamelCase					:						Dict					=			True
  
               if changes_to_apply:
                  UpperCamelCase					:						Optional[int]					=			getattr(SCREAMING_SNAKE_CASE_,    state.pop('type' ) )
                  UpperCamelCase					:						Tuple					=			component_class(**SCREAMING_SNAKE_CASE_ )
                  setattr(self.backend_tokenizer,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
         @property
         def       snake_case_ (    self )  ->	str:
            if self._mask_token is None:
               if self.verbose:
                  logger.error('Using mask_token, but it is not set yet.' )
               return None
            return str(self._mask_token )
         @mask_token.setter
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_ )  ->	List[str]:
            UpperCamelCase					:						Tuple					=			AddedToken(SCREAMING_SNAKE_CASE_,    lstrip=SCREAMING_SNAKE_CASE_,    rstrip=SCREAMING_SNAKE_CASE_ ) if isinstance(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ ) else value
            UpperCamelCase					:						str					=			value
         def       snake_case_ (    self,    *SCREAMING_SNAKE_CASE_,    **SCREAMING_SNAKE_CASE_ )  ->	BatchEncoding:
            UpperCamelCase					:						Dict					=			kwargs.get('is_split_into_words',    SCREAMING_SNAKE_CASE_ )
            assert self.add_prefix_space or not is_split_into_words, (
                F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                "to use it with pretokenized inputs."
            )
  
            return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE_,    **SCREAMING_SNAKE_CASE_ )
         def       snake_case_ (    self,    *SCREAMING_SNAKE_CASE_,    **SCREAMING_SNAKE_CASE_ )  ->	BatchEncoding:
            UpperCamelCase					:						List[Any]					=			kwargs.get('is_split_into_words',    SCREAMING_SNAKE_CASE_ )
  
            assert self.add_prefix_space or not is_split_into_words, (
                F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                "to use it with pretokenized inputs."
            )
  
            return super()._encode_plus(*SCREAMING_SNAKE_CASE_,    **SCREAMING_SNAKE_CASE_ )
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ = None )  ->	Tuple[str]:
            UpperCamelCase					:						Optional[Any]					=			self._tokenizer.model.save(SCREAMING_SNAKE_CASE_,    name=SCREAMING_SNAKE_CASE_ )
            return tuple(SCREAMING_SNAKE_CASE_ )
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=None )  ->	Optional[Any]:
            UpperCamelCase					:						Union[str, Any]					=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
            if token_ids_a is None:
               return output
  
            return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
  
  
  
  
  
  
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ = None )  ->	List[int]:
            UpperCamelCase					:						List[str]					=			[self.sep_token_id]
            UpperCamelCase					:						int					=			[self.cls_token_id]
  
            if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]
  
  
  
  
style_context: 
  
  
  
  import unittest
  
  import numpy as np
  
  from transformers.testing_utils import require_torch, require_vision
  from transformers.utils import is_torch_available, is_vision_available
  
  from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs
  
  
  if is_torch_available():
      import torch
  
  if is_vision_available():
      from PIL import Image
  
      from transformers import ViltImageProcessor
  
  
  
  class        lowerCAmelCase_	(						unittest.TestCase							):
         def __init__(    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=7,    SCREAMING_SNAKE_CASE_=3,    SCREAMING_SNAKE_CASE_=18,    SCREAMING_SNAKE_CASE_=30,    SCREAMING_SNAKE_CASE_=400,    SCREAMING_SNAKE_CASE_=True,    SCREAMING_SNAKE_CASE_=None,    SCREAMING_SNAKE_CASE_=2,    SCREAMING_SNAKE_CASE_=True,    SCREAMING_SNAKE_CASE_=[0.5, 0.5, 0.5],    SCREAMING_SNAKE_CASE_=[0.5, 0.5, 0.5],    )  ->	List[Any]:
            UpperCamelCase					:						Optional[int]					=			size if size is not None else {'shortest_edge': 30}
            UpperCamelCase					:						Dict					=			parent
            UpperCamelCase					:						str					=			batch_size
            UpperCamelCase					:						str					=			num_channels
            UpperCamelCase					:						Optional[int]					=			image_size
            UpperCamelCase					:						List[Any]					=			min_resolution
            UpperCamelCase					:						Tuple					=			max_resolution
            UpperCamelCase					:						Tuple					=			do_resize
            UpperCamelCase					:						Any					=			size
            UpperCamelCase					:						int					=			size_divisor
            UpperCamelCase					:						str					=			do_normalize
            UpperCamelCase					:						Optional[int]					=			image_mean
            UpperCamelCase					:						List[str]					=			image_std
         def       snake_case_ (    self )  ->	List[Any]:
            return {
                "image_mean": self.image_mean,
                "image_std": self.image_std,
                "do_normalize": self.do_normalize,
                "do_resize": self.do_resize,
                "size": self.size,
                "size_divisor": self.size_divisor,
            }
  
  
  
  
  
  
         def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=False )  ->	Union[str, Any]:
            if not batched:
               UpperCamelCase					:						Optional[Any]					=			self.size['shortest_edge']
               UpperCamelCase					:						int					=			image_inputs[0]
               if isinstance(SCREAMING_SNAKE_CASE_,    Image.Image ):
                  UpperCamelCase			,  UpperCamelCase					:						List[Any]					=			image.size
               else:
                  UpperCamelCase			,  UpperCamelCase					:						List[Any]					=			image.shape[1], image.shape[2]
               UpperCamelCase					:						Any					=			size / min(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
               if h < w:
                  UpperCamelCase			,  UpperCamelCase					:						Optional[int]					=			size, scale * w
               else:
                  UpperCamelCase			,  UpperCamelCase					:						List[str]					=			scale * h, size
  
               UpperCamelCase					:						int					=			int((1333 / 800) * size )
               if max(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ ) > max_size:
                  UpperCamelCase					:						Any					=			max_size / max(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )
                  UpperCamelCase					:						List[Any]					=			newh * scale
                  UpperCamelCase					:						Optional[int]					=			neww * scale
  
               UpperCamelCase			,  UpperCamelCase					:						List[Any]					=			int(newh + 0.5 ), int(neww + 0.5 )
               UpperCamelCase			,  UpperCamelCase					:						int					=			(
                   newh // self.size_divisor * self.size_divisor,
                   neww // self.size_divisor * self.size_divisor,
               )
  
            else:
               UpperCamelCase					:						Dict					=			[]
               for image in image_inputs:
                  UpperCamelCase			,  UpperCamelCase					:						Union[str, Any]					=			self.get_expected_values([image] )
                  expected_values.append((expected_height, expected_width) )
               UpperCamelCase					:						List[str]					=			max(SCREAMING_SNAKE_CASE_,    key=lambda SCREAMING_SNAKE_CASE_ : item[0] )[0]
               UpperCamelCase					:						List[str]					=			max(SCREAMING_SNAKE_CASE_,    key=lambda SCREAMING_SNAKE_CASE_ : item[1] )[1]
  
            return expected_height, expected_width
  
  
  
  
  
  
  @require_torch
  @require_vision
  class        lowerCAmelCase_	(						a__ ,					unittest.TestCase							):
         UpperCAmelCase__       :      List[Any]       =						ViltImageProcessor if is_vision_available() else None
         def       snake_case_ (    self )  ->	Union[str, Any]:
            UpperCamelCase					:						Dict					=			ViltImageProcessingTester(self )
         @property
         def       snake_case_ (    self )  ->	Dict:
            return self.image_processor_tester.prepare_image_processor_dict()
         def       snake_case_ (    self )  ->	Union[str, Any]:
            UpperCamelCase					:						List[str]					=			self.image_processing_class(**self.image_processor_dict )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'image_mean' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'image_std' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'do_normalize' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'do_resize' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'size' ) )
            self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_,    'size_divisor' ) )
         def       snake_case_ (    self )  ->	List[Any]:
            UpperCamelCase					:						Union[str, Any]					=			self.image_processing_class.from_dict(self.image_processor_dict )
            self.assertEqual(image_processor.size,    {'shortest_edge': 30} )
  
            UpperCamelCase					:						Union[str, Any]					=			self.image_processing_class.from_dict(self.image_processor_dict,    size=42 )
            self.assertEqual(image_processor.size,    {'shortest_edge': 42} )
         def       snake_case_ (    self )  ->	Dict:
            pass
         def       snake_case_ (    self )  ->	str:
            # Initialize image_processing
            UpperCamelCase					:						int					=			self.image_processing_class(**self.image_processor_dict )
            # create random PIL images
            UpperCamelCase					:						Any					=			prepare_image_inputs(self.image_processor_tester,    equal_resolution=SCREAMING_SNAKE_CASE_ )
            for image in image_inputs:
               self.assertIsInstance(SCREAMING_SNAKE_CASE_,    Image.Image )
  
            # Test not batched input
            UpperCamelCase					:						int					=			image_processing(image_inputs[0],    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Dict					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (1, self.image_processor_tester.num_channels, expected_height, expected_width),    )
  
            # Test batched
            UpperCamelCase					:						str					=			image_processing(SCREAMING_SNAKE_CASE_,    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Tuple					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_,    batched=SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    expected_height,
                    expected_width,
                ),    )
         def       snake_case_ (    self )  ->	List[str]:
            # Initialize image_processing
            UpperCamelCase					:						Optional[int]					=			self.image_processing_class(**self.image_processor_dict )
            # create random numpy tensors
            UpperCamelCase					:						Optional[Any]					=			prepare_image_inputs(self.image_processor_tester,    equal_resolution=SCREAMING_SNAKE_CASE_,    numpify=SCREAMING_SNAKE_CASE_ )
            for image in image_inputs:
               self.assertIsInstance(SCREAMING_SNAKE_CASE_,    np.ndarray )
  
            # Test not batched input
            UpperCamelCase					:						Optional[Any]					=			image_processing(image_inputs[0],    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						str					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (1, self.image_processor_tester.num_channels, expected_height, expected_width),    )
  
            # Test batched
            UpperCamelCase					:						Dict					=			image_processing(SCREAMING_SNAKE_CASE_,    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Optional[Any]					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_,    batched=SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    expected_height,
                    expected_width,
                ),    )
  
  
  
  
  
  
         def       snake_case_ (    self )  ->	Optional[Any]:
            # Initialize image_processing
            UpperCamelCase					:						Optional[int]					=			self.image_processing_class(**self.image_processor_dict )
            # create random PyTorch tensors
            UpperCamelCase					:						Dict					=			prepare_image_inputs(self.image_processor_tester,    equal_resolution=SCREAMING_SNAKE_CASE_,    torchify=SCREAMING_SNAKE_CASE_ )
            for image in image_inputs:
               self.assertIsInstance(SCREAMING_SNAKE_CASE_,    torch.Tensor )
  
            # Test not batched input
            UpperCamelCase					:						List[str]					=			image_processing(image_inputs[0],    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Union[str, Any]					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (1, self.image_processor_tester.num_channels, expected_height, expected_width),    )
  
            # Test batched
            UpperCamelCase					:						List[Any]					=			image_processing(SCREAMING_SNAKE_CASE_,    return_tensors='pt' ).pixel_values
  
            UpperCamelCase			,  UpperCamelCase					:						Dict					=			self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_,    batched=SCREAMING_SNAKE_CASE_ )
            self.assertEqual(
                encoded_images.shape,    (
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    expected_height,
                    expected_width,
                ),    )
  
  
  
  