code: 
  
  
  '''simple docstring'''
  
  import warnings
  from typing import List, Optional, Union
  
  from ...processing_utils import ProcessorMixin
  from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
  from ...utils import TensorType
  class _lowerCamelCase (		lowercase__							):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		A_			:    str   					=       ["""image_processor""", """tokenizer"""]
  		A_			:    int   					=       """LayoutLMv2ImageProcessor"""
  		A_			:    Optional[Any]   					=       ("""LayoutLMv2Tokenizer""", """LayoutLMv2TokenizerFast""")
  
  
  
  
  
  
  
  		def __init__(      self	:		Any						,  _A	:		int=None						,  _A	:		Dict=None						,  **_A	:		Tuple    )						->				str:
  			__magic_name__ :					Optional[Any]  			=  None
  			if "feature_extractor" in kwargs:
  				warnings.warn(
  				    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
  				    ' instead.'						,  _A						,  )
  				__magic_name__ :					str  			=  kwargs.pop('feature_extractor'    )
  
  			__magic_name__ :					Optional[Any]  			=  image_processor if image_processor is not None else feature_extractor
  			if image_processor is None:
  				raise ValueError('You need to specify an `image_processor`.'    )
  			if tokenizer is None:
  				raise ValueError('You need to specify a `tokenizer`.'    )
  
  			super().__init__(_A						,  _A    )
  
  
  
  
  
  
  
  		def __call__(      self	:		Optional[int]						,  _A	:		Union[str, Any]						,  _A	:		Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None						,  _A	:		Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None						,  _A	:		Union[List[List[int]], List[List[List[int]]]] = None						,  _A	:		Optional[Union[List[int], List[List[int]]]] = None						,  _A	:		bool = True						,  _A	:		Union[bool, str, PaddingStrategy] = False						,  _A	:		Union[bool, str, TruncationStrategy] = False						,  _A	:		Optional[int] = None						,  _A	:		int = 0						,  _A	:		Optional[int] = None						,  _A	:		Optional[bool] = None						,  _A	:		Optional[bool] = None						,  _A	:		bool = False						,  _A	:		bool = False						,  _A	:		bool = False						,  _A	:		bool = False						,  _A	:		bool = True						,  _A	:		Optional[Union[str, TensorType]] = None						,  **_A	:		Any						,  )						->				BatchEncoding:
  			# verify input
  			if self.image_processor.apply_ocr and (boxes is not None):
  				raise ValueError(
  				    'You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True.'    )
  
  			if self.image_processor.apply_ocr and (word_labels is not None):
  				raise ValueError(
  				    'You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'    )
  
  			if return_overflowing_tokens is True and return_offsets_mapping is False:
  				raise ValueError('You cannot return overflowing tokens without returning the offsets mapping.'    )
  
  			# first, apply the image processor
  			__magic_name__ :					str  			=  self.image_processor(images=_A						,  return_tensors=_A    )
  
  			# second, apply the tokenizer
  			if text is not None and self.image_processor.apply_ocr and text_pair is None:
  				if isinstance(_A						,  _A    ):
  					__magic_name__ :					Union[str, Any]  			=  [text]  # add batch dimension (as the image processor always adds a batch dimension)
  				__magic_name__ :					Tuple  			=  features['words']
  
  			__magic_name__ :					Optional[Any]  			=  self.tokenizer(
  			    text=text if text is not None else features['words']						,  text_pair=text_pair if text_pair is not None else None						,  boxes=boxes if boxes is not None else features['boxes']						,  word_labels=_A						,  add_special_tokens=_A						,  padding=_A						,  truncation=_A						,  max_length=_A						,  stride=_A						,  pad_to_multiple_of=_A						,  return_token_type_ids=_A						,  return_attention_mask=_A						,  return_overflowing_tokens=_A						,  return_special_tokens_mask=_A						,  return_offsets_mapping=_A						,  return_length=_A						,  verbose=_A						,  return_tensors=_A						,  **_A						,  )
  
  			# add pixel values
  			__magic_name__ :					Union[str, Any]  			=  features.pop('pixel_values'    )
  			if return_overflowing_tokens is True:
  				__magic_name__ :					List[str]  			=  self.get_overflowing_images(_A						,  encoded_inputs['overflow_to_sample_mapping']    )
  			__magic_name__ :					Optional[Any]  			=  images
  
  			return encoded_inputs
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		Optional[Any]						,  _A	:		Union[str, Any]						,  _A	:		Tuple    )						->				int:
  			# in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
  			__magic_name__ :					Dict  			=  []
  			for sample_idx in overflow_to_sample_mapping:
  				images_with_overflow.append(images[sample_idx]    )
  
  			if len(_A    ) != len(_A    ):
  				raise ValueError(
  				    'Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'
  				    F' {len(_A    )} and {len(_A    )}'    )
  
  			return images_with_overflow
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		Union[str, Any]						,  *_A	:		Tuple						,  **_A	:		int    )						->				Dict:
  			return self.tokenizer.batch_decode(*_A						,  **_A    )
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		List[Any]						,  *_A	:		Optional[int]						,  **_A	:		Optional[int]    )						->				List[Any]:
  			return self.tokenizer.decode(*_A						,  **_A    )
  
  
  
  
  
  
  
  		@property
  		def      __lowerCAmelCase	(      self	:		int    )						->				Any:
  			return ["input_ids", "bbox", "token_type_ids", "attention_mask", "image"]
  
  
  
  
  
  
  
  		@property
  		def      __lowerCAmelCase	(      self	:		List[str]    )						->				Optional[Any]:
  			warnings.warn(
  			    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'						,  _A						,  )
  			return self.image_processor_class
  
  
  
  
  
  
  
  		@property
  		def      __lowerCAmelCase	(      self	:		Optional[Any]    )						->				List[Any]:
  			warnings.warn(
  			    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'						,  _A						,  )
  			return self.image_processor
style_context: 
  
  
  '''simple docstring'''
  
  import os
  import pickle
  import tempfile
  import unittest
  from typing import Callable, Optional
  
  import numpy as np
  
  from transformers import (
      BatchEncoding,
      BertTokenizer,
      BertTokenizerFast,
      PreTrainedTokenizer,
      PreTrainedTokenizerFast,
      TensorType,
      TokenSpan,
      is_tokenizers_available,
  )
  from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
  from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
  
  
  if is_tokenizers_available():
  							from tokenizers import Tokenizer
  							from tokenizers.models import WordPiece
  class _lowerCamelCase (		unittest.TestCase							):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		Any						,  _A	:		Optional[Any]    )						->				Optional[int]:
  			__magic_name__ :					Optional[int]  			=  list(tokenizer_class.max_model_input_sizes.keys()    )
  			for model_name in sa_models[:1]:
  				__magic_name__ :					List[str]  			=  tokenizer_class.from_pretrained(_A    )
  				self.assertIsNotNone(_A    )
  				self.assertIsInstance(_A						,  _A    )
  				self.assertIsInstance(_A						,  _A    )
  
  				for special_tok in tokenizer.all_special_tokens:
  					self.assertIsInstance(_A						,  _A    )
  					__magic_name__ :					Optional[Any]  			=  tokenizer.convert_tokens_to_ids(_A    )
  					self.assertIsInstance(_A						,  _A    )
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		Optional[Any]						,  _A	:		BatchEncoding						,  _A	:		Optional[Callable] = None    )						->				Tuple:
  			__magic_name__ :					Union[str, Any]  			=  pickle.dumps(_A    )
  			self.assertIsNotNone(_A    )
  
  			__magic_name__ :					Optional[Any]  			=  pickle.loads(_A    )
  
  			# Ensure is_fast is correctly restored
  			self.assertEqual(be_restored.is_fast						,  be_original.is_fast    )
  
  			# Ensure encodings are potentially correctly restored
  			if be_original.is_fast:
  				self.assertIsNotNone(be_restored.encodings    )
  			else:
  				self.assertIsNone(be_restored.encodings    )
  
  			# Ensure the keys are the same
  			for original_v, restored_v in zip(be_original.values()						,  be_restored.values()    ):
  				if equal_op:
  					self.assertTrue(equal_op(_A						,  _A    )    )
  				else:
  					self.assertEqual(_A						,  _A    )
  
  
  
  
  
  
  
  		@slow
  		def      __lowerCAmelCase	(      self	:		Dict    )						->				Optional[Any]:
  			self.check_tokenizer_from_pretrained(_A    )
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		str    )						->				List[Any]:
  			self.assertEqual(TensorType('tf'    )						,  TensorType.TENSORFLOW    )
  			self.assertEqual(TensorType('pt'    )						,  TensorType.PYTORCH    )
  			self.assertEqual(TensorType('np'    )						,  TensorType.NUMPY    )
  
  
  
  
  
  
  
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		List[str]    )						->				Optional[int]:
  			import numpy as np
  
  			__magic_name__ :					Dict  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  			__magic_name__ :					str  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )
  
  			# Python no tensor
  			with self.subTest('BatchEncoding (Python, return_tensors=None)'    ):
  				self.assert_dump_and_restore(tokenizer_p('Small example to encode'    )    )
  
  			with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'    ):
  				self.assert_dump_and_restore(
  				    tokenizer_p('Small example to encode'						,  return_tensors=TensorType.NUMPY    )						,  np.array_equal    )
  
  			with self.subTest('BatchEncoding (Rust, return_tensors=None)'    ):
  				self.assert_dump_and_restore(tokenizer_r('Small example to encode'    )    )
  
  			with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'    ):
  				self.assert_dump_and_restore(
  				    tokenizer_r('Small example to encode'						,  return_tensors=TensorType.NUMPY    )						,  np.array_equal    )
  
  
  
  
  
  
  
  		@require_tf
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		List[Any]    )						->				List[str]:
  			import tensorflow as tf
  
  			def tf_array_equals(_A	:		str						,  _A	:		Union[str, Any]    ):
  				return tf.reduce_all(tf.equal(_A						,  _A    )    )
  
  			__magic_name__ :					str  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  			__magic_name__ :					Optional[Any]  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )
  
  			with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'    ):
  				self.assert_dump_and_restore(
  				    tokenizer_p('Small example to encode'						,  return_tensors=TensorType.TENSORFLOW    )						,  _A    )
  
  			with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'    ):
  				self.assert_dump_and_restore(
  				    tokenizer_r('Small example to encode'						,  return_tensors=TensorType.TENSORFLOW    )						,  _A    )
  
  
  
  
  
  
  
  		@require_torch
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		Any    )						->				List[str]:
  			import torch
  
  			__magic_name__ :					Union[str, Any]  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  			__magic_name__ :					Optional[int]  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )
  
  			with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'    ):
  				self.assert_dump_and_restore(
  				    tokenizer_p('Small example to encode'						,  return_tensors=TensorType.PYTORCH    )						,  torch.equal    )
  
  			with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'    ):
  				self.assert_dump_and_restore(
  				    tokenizer_r('Small example to encode'						,  return_tensors=TensorType.PYTORCH    )						,  torch.equal    )
  
  
  
  
  
  
  
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		List[str]    )						->				Dict:
  			__magic_name__ :					Optional[int]  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  			__magic_name__ :					Dict  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )
  
  			with self.subTest('Python Tokenizer'    ):
  				self.assertFalse(tokenizer_p('Small example to_encode'    ).is_fast    )
  
  			with self.subTest('Rust Tokenizer'    ):
  				self.assertTrue(tokenizer_r('Small example to_encode'    ).is_fast    )
  
  
  
  
  
  
  
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		Union[str, Any]    )						->				Optional[int]:
  			__magic_name__ :					Optional[int]  			=  BertTokenizerFast.from_pretrained('bert-base-cased'    )
  			__magic_name__ :					Tuple  			=  tokenizer_r(['Test', '\xad', 'test']						,  is_split_into_words=_A    )
  
  			self.assertEqual(encoded.word_to_tokens(0    )						,  TokenSpan(start=1						,  end=2    )    )
  			self.assertEqual(encoded.word_to_tokens(1    )						,  _A    )
  			self.assertEqual(encoded.word_to_tokens(2    )						,  TokenSpan(start=2						,  end=3    )    )
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		Optional[int]    )						->				Optional[Any]:
  			__magic_name__ :					Any  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
  			__magic_name__ :					Dict  			=  batch.convert_to_tensors(tensor_type='np'    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
  			# test converting the converted
  			with CaptureStderr() as cs:
  				__magic_name__ :					Any  			=  batch.convert_to_tensors(tensor_type='np'    )
  			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )
  
  			__magic_name__ :					Optional[Any]  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
  			__magic_name__ :					List[Any]  			=  batch.convert_to_tensors(tensor_type='np'						,  prepend_batch_axis=_A    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )
  
  
  
  
  
  
  
  		@require_torch
  		def      __lowerCAmelCase	(      self	:		Union[str, Any]    )						->				Any:
  			__magic_name__ :					Optional[int]  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
  			__magic_name__ :					List[Any]  			=  batch.convert_to_tensors(tensor_type='pt'    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
  			# test converting the converted
  			with CaptureStderr() as cs:
  				__magic_name__ :					Union[str, Any]  			=  batch.convert_to_tensors(tensor_type='pt'    )
  			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )
  
  			__magic_name__ :					Any  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
  			__magic_name__ :					str  			=  batch.convert_to_tensors(tensor_type='pt'						,  prepend_batch_axis=_A    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )
  
  
  
  
  
  
  
  		@require_tf
  		def      __lowerCAmelCase	(      self	:		Optional[Any]    )						->				List[str]:
  			__magic_name__ :					str  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
  			__magic_name__ :					List[str]  			=  batch.convert_to_tensors(tensor_type='tf'    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
  			# test converting the converted
  			with CaptureStderr() as cs:
  				__magic_name__ :					Optional[int]  			=  batch.convert_to_tensors(tensor_type='tf'    )
  			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )
  
  			__magic_name__ :					Union[str, Any]  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
  			__magic_name__ :					Union[str, Any]  			=  batch.convert_to_tensors(tensor_type='tf'						,  prepend_batch_axis=_A    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )
  
  
  
  
  
  
  
  		@require_flax
  		def      __lowerCAmelCase	(      self	:		str    )						->				Optional[Any]:
  			__magic_name__ :					List[Any]  			=  BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}    )
  			__magic_name__ :					int  			=  batch.convert_to_tensors(tensor_type='jax'    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (2, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (2,)    )
  			# test converting the converted
  			with CaptureStderr() as cs:
  				__magic_name__ :					Optional[Any]  			=  batch.convert_to_tensors(tensor_type='jax'    )
  			self.assertFalse(len(cs.err    )						,  msg=F'should have no warning, but got {cs.err}'    )
  
  			__magic_name__ :					str  			=  BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}    )
  			__magic_name__ :					Dict  			=  batch.convert_to_tensors(tensor_type='jax'						,  prepend_batch_axis=_A    )
  			self.assertEqual(tensor_batch['inputs'].shape						,  (1, 3)    )
  			self.assertEqual(tensor_batch['labels'].shape						,  (1,)    )
  
  
  
  
  
  
  
  		def      __lowerCAmelCase	(      self	:		str    )						->				int:
  			__magic_name__ :					int  			=  [{'input_ids': np.array([0, 1, 2]    )}, {'input_ids': np.array([0, 1, 2, 3]    )}]
  			__magic_name__ :					Dict  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  
  			__magic_name__ :					Optional[int]  			=  tokenizer.pad(_A						,  padding=_A    )
  			self.assertTrue(isinstance(batch['input_ids']						,  np.ndarray    )    )
  			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
  			__magic_name__ :					Dict  			=  tokenizer.pad(_A						,  padding=_A						,  return_tensors='np'    )
  			self.assertTrue(isinstance(batch['input_ids']						,  np.ndarray    )    )
  			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
  
  
  
  
  
  
  
  		@require_torch
  		def      __lowerCAmelCase	(      self	:		List[Any]    )						->				str:
  			import torch
  
  			__magic_name__ :					List[str]  			=  [{'input_ids': torch.tensor([0, 1, 2]    )}, {'input_ids': torch.tensor([0, 1, 2, 3]    )}]
  			__magic_name__ :					List[str]  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  
  			__magic_name__ :					List[Any]  			=  tokenizer.pad(_A						,  padding=_A    )
  			self.assertTrue(isinstance(batch['input_ids']						,  torch.Tensor    )    )
  			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
  			__magic_name__ :					Union[str, Any]  			=  tokenizer.pad(_A						,  padding=_A						,  return_tensors='pt'    )
  			self.assertTrue(isinstance(batch['input_ids']						,  torch.Tensor    )    )
  			self.assertEqual(batch['input_ids'].tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
  
  
  
  
  
  
  
  		@require_tf
  		def      __lowerCAmelCase	(      self	:		Any    )						->				Optional[int]:
  			import tensorflow as tf
  
  			__magic_name__ :					Tuple  			=  [{'input_ids': tf.constant([0, 1, 2]    )}, {'input_ids': tf.constant([0, 1, 2, 3]    )}]
  			__magic_name__ :					Any  			=  BertTokenizer.from_pretrained('bert-base-cased'    )
  
  			__magic_name__ :					Any  			=  tokenizer.pad(_A						,  padding=_A    )
  			self.assertTrue(isinstance(batch['input_ids']						,  tf.Tensor    )    )
  			self.assertEqual(batch['input_ids'].numpy().tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
  			__magic_name__ :					Union[str, Any]  			=  tokenizer.pad(_A						,  padding=_A						,  return_tensors='tf'    )
  			self.assertTrue(isinstance(batch['input_ids']						,  tf.Tensor    )    )
  			self.assertEqual(batch['input_ids'].numpy().tolist()						,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
  
  
  
  
  
  
  
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		Any    )						->				str:
  			__magic_name__ :					str  			=  Tokenizer(WordPiece(unk_token='[UNK]'    )    )
  			PreTrainedTokenizerFast(tokenizer_object=_A    )
  
  
  
  
  
  
  
  		@require_tokenizers
  		def      __lowerCAmelCase	(      self	:		Dict    )						->				Any:
  			__magic_name__ :					List[Any]  			=  Tokenizer(WordPiece(unk_token='[UNK]'    )    )
  			with tempfile.TemporaryDirectory() as tmpdirname:
  				bert_tokenizer.save(os.path.join(_A						,  'tokenizer.json'    )    )
  				PreTrainedTokenizerFast(tokenizer_file=os.path.join(_A						,  'tokenizer.json'    )    )