

import pytest

import datasets.config
from datasets.utils.info_utils import is_small_dataset

@pytest.mark.parametrize('dataset_size'	,     [None, 400 * 2**20, 600 * 2**20]			)
@pytest.mark.parametrize('input_in_memory_max_size'	,     ['default', 0, 100 * 2**20, 900 * 2**20]			)
def 	snake_case__			(    SCREAMING_SNAKE_CASE_		:    List[str]	,     SCREAMING_SNAKE_CASE_		:    str	,     SCREAMING_SNAKE_CASE_		:    List[Any]			):



     '''simple docstring'''





     if input_in_memory_max_size != "default":
          monkeypatch.setattr(datasets.config	,     'IN_MEMORY_MAX_SIZE'	,     SCREAMING_SNAKE_CASE_			)
     lowercase__    :   List[Any]      =       datasets.config.IN_MEMORY_MAX_SIZE
     if input_in_memory_max_size == "default":
          assert in_memory_max_size == 0
     else:
          assert in_memory_max_size == input_in_memory_max_size
     if dataset_size and in_memory_max_size:
          lowercase__    :   Dict      =       dataset_size < in_memory_max_size
     else:
          lowercase__    :   int      =       False
     lowercase__    :   Optional[int]      =       is_small_dataset(SCREAMING_SNAKE_CASE_			)
     assert result == expected




from __future__ import annotations

import copy
import inspect
import unittest

import numpy as np

from transformers import is_tf_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers import (
           TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
           TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
           TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
           TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
           LayoutLMvaConfig,
           TFLayoutLMvaForQuestionAnswering,
           TFLayoutLMvaForSequenceClassification,
           TFLayoutLMvaForTokenClassification,
           TFLayoutLMvaModel,
       )

if is_vision_available():
       from PIL import Image

       from transformers import LayoutLMvaImageProcessor


class       SCREAMING_SNAKE_CASE__					:





    def __init__(      self						,   a						,   a=2						,   a=3						,   a=4						,   a=2						,   a=7						,   a=True						,   a=True						,   a=True						,   a=True						,   a=99						,   a=36						,   a=2						,   a=4						,   a=37						,   a="gelu"						,   a=0.1						,   a=0.1						,   a=512						,   a=16						,   a=2						,   a=0.02						,   a=6						,   a=6						,   a=3						,   a=4						,   a=None						,   a=1000						,   ):
         lowercase__    :   List[str]      =       parent
         lowercase__    :   List[str]      =       batch_size
         lowercase__    :   int      =       num_channels
         lowercase__    :   List[Any]      =       image_size
         lowercase__    :   List[str]      =       patch_size
         lowercase__    :   List[Any]      =       is_training
         lowercase__    :   Tuple      =       use_input_mask
         lowercase__    :   str      =       use_token_type_ids
         lowercase__    :   Optional[int]      =       use_labels
         lowercase__    :   Any      =       vocab_size
         lowercase__    :   List[str]      =       hidden_size
         lowercase__    :   Union[str, Any]      =       num_hidden_layers
         lowercase__    :   List[Any]      =       num_attention_heads
         lowercase__    :   Dict      =       intermediate_size
         lowercase__    :   int      =       hidden_act
         lowercase__    :   Optional[Any]      =       hidden_dropout_prob
         lowercase__    :   int      =       attention_probs_dropout_prob
         lowercase__    :   str      =       max_position_embeddings
         lowercase__    :   List[Any]      =       type_vocab_size
         lowercase__    :   str      =       type_sequence_label_size
         lowercase__    :   List[Any]      =       initializer_range
         lowercase__    :   Union[str, Any]      =       coordinate_size
         lowercase__    :   Union[str, Any]      =       shape_size
         lowercase__    :   Any      =       num_labels
         lowercase__    :   List[str]      =       num_choices
         lowercase__    :   Optional[Any]      =       scope
         lowercase__    :   Optional[Any]      =       range_bbox

         # LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
         lowercase__    :   Optional[int]      =       text_seq_length
         lowercase__    :   Optional[int]      =       (image_size // patch_size) ** 2 + 1
         lowercase__    :   str      =       self.text_seq_length + self.image_seq_length





    def       snake_case_				(      self):
         lowercase__    :   Dict      =       ids_tensor([self.batch_size, self.text_seq_length]						,   self.vocab_size)

         lowercase__    :   List[Any]      =       ids_tensor([self.batch_size, self.text_seq_length, 4]						,   self.range_bbox)
         lowercase__    :   str      =       bbox.numpy()
         # Ensure that bbox is legal
         for i in range(bbox.shape[0]):
              for j in range(bbox.shape[1]):
                   if bbox[i, j, 3] < bbox[i, j, 1]:
                        lowercase__    :   Optional[Any]      =       bbox[i, j, 3]
                        lowercase__    :   List[Any]      =       bbox[i, j, 1]
                        lowercase__    :   List[str]      =       tmp_coordinate
                   if bbox[i, j, 2] < bbox[i, j, 0]:
                        lowercase__    :   int      =       bbox[i, j, 2]
                        lowercase__    :   List[Any]      =       bbox[i, j, 0]
                        lowercase__    :   Optional[Any]      =       tmp_coordinate
         lowercase__    :   Dict      =       tf.constant(a)

         lowercase__    :   Dict      =       floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

         lowercase__    :   Optional[Any]      =       None
         if self.use_input_mask:
              lowercase__    :   str      =       random_attention_mask([self.batch_size, self.text_seq_length])

         lowercase__    :   Tuple      =       None
         if self.use_token_type_ids:
              lowercase__    :   str      =       ids_tensor([self.batch_size, self.text_seq_length]						,   self.type_vocab_size)

         lowercase__    :   List[Any]      =       None
         lowercase__    :   Optional[int]      =       None
         if self.use_labels:
              lowercase__    :   Tuple      =       ids_tensor([self.batch_size]						,   self.type_sequence_label_size)
              lowercase__    :   Union[str, Any]      =       ids_tensor([self.batch_size, self.text_seq_length]						,   self.num_labels)

         lowercase__    :   List[str]      =       LayoutLMvaConfig(
             vocab_size=self.vocab_size						,   hidden_size=self.hidden_size						,   num_hidden_layers=self.num_hidden_layers						,   num_attention_heads=self.num_attention_heads						,   intermediate_size=self.intermediate_size						,   hidden_act=self.hidden_act						,   hidden_dropout_prob=self.hidden_dropout_prob						,   attention_probs_dropout_prob=self.attention_probs_dropout_prob						,   max_position_embeddings=self.max_position_embeddings						,   type_vocab_size=self.type_vocab_size						,   initializer_range=self.initializer_range						,   coordinate_size=self.coordinate_size						,   shape_size=self.shape_size						,   input_size=self.image_size						,   patch_size=self.patch_size						,   )

         return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels





    def       snake_case_				(      self						,   a						,   a						,   a						,   a						,   a						,   a):
         lowercase__    :   str      =       TFLayoutLMvaModel(config=a)

         # text + image
         lowercase__    :   List[str]      =       model(a						,   pixel_values=a						,   training=a)
         lowercase__    :   Any      =       model(
             a						,   bbox=a						,   pixel_values=a						,   attention_mask=a						,   token_type_ids=a						,   training=a						,   )
         lowercase__    :   List[Any]      =       model(a						,   bbox=a						,   pixel_values=a						,   training=a)

         self.parent.assertEqual(result.last_hidden_state.shape						,   (self.batch_size, self.seq_length, self.hidden_size))

         # text only
         lowercase__    :   List[Any]      =       model(a						,   training=a)
         self.parent.assertEqual(
             result.last_hidden_state.shape						,   (self.batch_size, self.text_seq_length, self.hidden_size))

         # image only
         lowercase__    :   Dict      =       model({'pixel_values': pixel_values}						,   training=a)
         self.parent.assertEqual(
             result.last_hidden_state.shape						,   (self.batch_size, self.image_seq_length, self.hidden_size))





    def       snake_case_				(      self						,   a						,   a						,   a						,   a						,   a						,   a						,   a):
         lowercase__    :   Optional[Any]      =       self.num_labels
         lowercase__    :   Optional[Any]      =       TFLayoutLMvaForSequenceClassification(config=a)
         lowercase__    :   List[str]      =       model(
             a						,   bbox=a						,   pixel_values=a						,   attention_mask=a						,   token_type_ids=a						,   labels=a						,   training=a						,   )
         self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.num_labels))





    def       snake_case_				(      self						,   a						,   a						,   a						,   a						,   a						,   a						,   a):
         lowercase__    :   Tuple      =       self.num_labels
         lowercase__    :   Dict      =       TFLayoutLMvaForTokenClassification(config=a)
         lowercase__    :   Any      =       model(
             a						,   bbox=a						,   pixel_values=a						,   attention_mask=a						,   token_type_ids=a						,   labels=a						,   training=a						,   )
         self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.text_seq_length, self.num_labels))





    def       snake_case_				(      self						,   a						,   a						,   a						,   a						,   a						,   a						,   a):
         lowercase__    :   Optional[int]      =       2
         lowercase__    :   List[str]      =       TFLayoutLMvaForQuestionAnswering(config=a)
         lowercase__    :   Tuple      =       model(
             a						,   bbox=a						,   pixel_values=a						,   attention_mask=a						,   token_type_ids=a						,   start_positions=a						,   end_positions=a						,   training=a						,   )
         self.parent.assertEqual(result.start_logits.shape						,   (self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape						,   (self.batch_size, self.seq_length))





    def       snake_case_				(      self):
         lowercase__    :   Union[str, Any]      =       self.prepare_config_and_inputs()
         ((lowercase__)      , (lowercase__)      , (lowercase__)      , (lowercase__)      , (lowercase__)      , (lowercase__)      , (lowercase__)      , (lowercase__))    :   Optional[Any]      =       config_and_inputs
         lowercase__    :   Optional[Any]      =       {
             'input_ids': input_ids,
             'bbox': bbox,
             'pixel_values': pixel_values,
             'token_type_ids': token_type_ids,
             'attention_mask': input_mask,
         }
         return config, inputs_dict



@require_tf
class       SCREAMING_SNAKE_CASE__					(__snake_case		,       __snake_case		,       unittest.TestCase     ):
    __lowerCamelCase						:       List[str]														=     (
        (
            TFLayoutLMvaModel,
            TFLayoutLMvaForQuestionAnswering,
            TFLayoutLMvaForSequenceClassification,
            TFLayoutLMvaForTokenClassification,
        )
        if is_tf_available()
        else ()
    )
    __lowerCamelCase						:       Dict														=     (
        {"""document-question-answering""": TFLayoutLMvaForQuestionAnswering, """feature-extraction""": TFLayoutLMvaModel}
        if is_tf_available()
        else {}
    )

    __lowerCamelCase						:       Optional[Any]														=     False
    __lowerCamelCase						:       int														=     False
    __lowerCamelCase						:       int														=     False





    def       snake_case_				(      self						,   a						,   a						,   a						,   a						,   a):
         return True





    def       snake_case_				(      self						,   a						,   a						,   a=False):
         lowercase__    :   Tuple      =       copy.deepcopy(a)

         if model_class in get_values(a):
              lowercase__    :   Optional[Any]      =       {
                  k: tf.tile(tf.expand_dims(a						,   1)						,   (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1))
                  if isinstance(a						,   tf.Tensor) and v.ndim > 0
                  else v
                  for k, v in inputs_dict.items()
              }

         if return_labels:
              if model_class in get_values(a):
                   lowercase__    :   Union[str, Any]      =       tf.ones(self.model_tester.batch_size						,   dtype=tf.intaa)
              elif model_class in get_values(a):
                   lowercase__    :   Dict      =       tf.zeros(self.model_tester.batch_size						,   dtype=tf.intaa)
                   lowercase__    :   Optional[int]      =       tf.zeros(self.model_tester.batch_size						,   dtype=tf.intaa)
              elif model_class in get_values(a):
                   lowercase__    :   Optional[Any]      =       tf.zeros(self.model_tester.batch_size						,   dtype=tf.intaa)
              elif model_class in get_values(a):
                   lowercase__    :   Optional[int]      =       tf.zeros(
                       (self.model_tester.batch_size, self.model_tester.text_seq_length)						,   dtype=tf.intaa)

         return inputs_dict





    def       snake_case_				(      self):
         lowercase__    :   Tuple      =       TFLayoutLMvaModelTester(self)
         lowercase__    :   Optional[Any]      =       ConfigTester(self						,   config_class=a						,   hidden_size=37)





    def       snake_case_				(      self):
         self.config_tester.run_common_tests()





    def       snake_case_				(      self):
         lowercase__      , lowercase__    :   List[Any]      =       self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
              lowercase__    :   List[Any]      =       model_class(a)
              if getattr(a						,   'hf_compute_loss'						,   a):
                   # The number of elements in the loss should be the same as the number of elements in the label
                   lowercase__    :   Optional[int]      =       self._prepare_for_class(inputs_dict.copy()						,   a						,   return_labels=a)
                   lowercase__    :   Union[str, Any]      =       prepared_for_class[
                       sorted(prepared_for_class.keys() - inputs_dict.keys()						,   reverse=a)[0]
                   ]
                   lowercase__    :   Tuple      =       added_label.shape.as_list()[:1]

                   # Test that model correctly compute the loss with kwargs
                   lowercase__    :   Dict      =       self._prepare_for_class(inputs_dict.copy()						,   a						,   return_labels=a)
                   lowercase__    :   int      =       prepared_for_class.pop('input_ids')

                   lowercase__    :   Optional[int]      =       model(a						,   **a)[0]
                   self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])

                   # Test that model correctly compute the loss when we mask some positions
                   lowercase__    :   str      =       self._prepare_for_class(inputs_dict.copy()						,   a						,   return_labels=a)
                   lowercase__    :   str      =       prepared_for_class.pop('input_ids')
                   if "labels" in prepared_for_class:
                        lowercase__    :   Optional[Any]      =       prepared_for_class['labels'].numpy()
                        if len(labels.shape) > 1 and labels.shape[1] != 1:
                             lowercase__    :   Union[str, Any]      =       -100
                             lowercase__    :   Optional[Any]      =       tf.convert_to_tensor(a)
                             lowercase__    :   Any      =       model(a						,   **a)[0]
                             self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])
                             self.assertTrue(not np.any(np.isnan(loss.numpy())))

                # Test that model correctly compute the loss with a dict
                   lowercase__    :   List[Any]      =       self._prepare_for_class(inputs_dict.copy()						,   a						,   return_labels=a)
                   lowercase__    :   Optional[Any]      =       model(a)[0]
                   self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])

                   # Test that model correctly compute the loss with a tuple
                   lowercase__    :   List[str]      =       self._prepare_for_class(inputs_dict.copy()						,   a						,   return_labels=a)

                   # Get keys that were added with the _prepare_for_class function
                   lowercase__    :   int      =       prepared_for_class.keys() - inputs_dict.keys()
                   lowercase__    :   List[Any]      =       inspect.signature(model.call).parameters
                   lowercase__    :   List[str]      =       list(signature.keys())

                   # Create a dictionary holding the location of the tensors in the tuple
                   lowercase__    :   Dict      =       {0: 'input_ids'}
                   for label_key in label_keys:
                        lowercase__    :   Tuple      =       signature_names.index(a)
                        lowercase__    :   List[str]      =       label_key
                   lowercase__    :   int      =       sorted(tuple_index_mapping.items())
                   # Initialize a list with their default values, update the values and convert to a tuple
                   lowercase__    :   List[Any]      =       []

                   for name in signature_names:
                        if name != "kwargs":
                             list_input.append(signature[name].default)

                   for index, value in sorted_tuple_index_mapping:
                        lowercase__    :   Optional[int]      =       prepared_for_class[value]

                   lowercase__    :   Any      =       tuple(a)

                   # Send to model
                   lowercase__    :   List[str]      =       model(tuple_input[:-1])[0]

                   self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])





    def       snake_case_				(      self):
         (
             (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , 
         )    :   List[Any]      =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(a						,   a						,   a						,   a						,   a						,   a)





    def       snake_case_				(      self):
         (
             (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , 
         )    :   int      =       self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
              lowercase__    :   Dict      =       type
              self.model_tester.create_and_check_model(a						,   a						,   a						,   a						,   a						,   a)





    def       snake_case_				(      self):
         (
             (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , 
         )    :   Tuple      =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(
             a						,   a						,   a						,   a						,   a						,   a						,   a)





    def       snake_case_				(      self):
         (
             (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , 
         )    :   Dict      =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(
             a						,   a						,   a						,   a						,   a						,   a						,   a)





    def       snake_case_				(      self):
         (
             (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , (
             lowercase__
         )      , 
         )    :   List[str]      =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(
             a						,   a						,   a						,   a						,   a						,   a						,   a)





    @slow
    def       snake_case_				(      self):
         for model_name in TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              lowercase__    :   str      =       TFLayoutLMvaModel.from_pretrained(a)
              self.assertIsNotNone(a)



def 	snake_case__			(    ):



     '''simple docstring'''





     lowercase__    :   str      =       Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'			)
     return image


@require_tf
class       SCREAMING_SNAKE_CASE__					(unittest.TestCase     ):





    @cached_property
    def       snake_case_				(      self):
         return LayoutLMvaImageProcessor(apply_ocr=a) if is_vision_available() else None





    @slow
    def       snake_case_				(      self):
         lowercase__    :   Optional[int]      =       TFLayoutLMvaModel.from_pretrained('microsoft/layoutlmv3-base')

         lowercase__    :   Tuple      =       self.default_image_processor
         lowercase__    :   Union[str, Any]      =       prepare_img()
         lowercase__    :   Optional[int]      =       image_processor(images=a						,   return_tensors='tf').pixel_values

         lowercase__    :   List[Any]      =       tf.constant([[1, 2]])
         lowercase__    :   str      =       tf.expand_dims(tf.constant([[1, 2, 3, 4], [5, 6, 7, 8]])						,   axis=0)

         # forward pass
         lowercase__    :   List[str]      =       model(input_ids=a						,   bbox=a						,   pixel_values=a						,   training=a)

         # verify the logits
         lowercase__    :   Optional[int]      =       (1, 199, 768)
         self.assertEqual(outputs.last_hidden_state.shape						,   a)

         lowercase__    :   Union[str, Any]      =       tf.constant(
             [[-0.0_529, 0.3_618, 0.1_632], [-0.1_587, -0.1_667, -0.0_400], [-0.1_557, -0.1_671, -0.0_505]])

         self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]						,   a						,   atol=1e-4))

