





'''simple docstring'''




from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers.models.layoutlm.modeling_tf_layoutlm import (
      TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFLayoutLMForMaskedLM,
      TFLayoutLMForQuestionAnswering,
      TFLayoutLMForSequenceClassification,
      TFLayoutLMForTokenClassification,
      TFLayoutLMModel,
  )


class  SCREAMING_SNAKE_CASE    :


     def __init__(						self						,	_UpperCAmelCase						,	_UpperCAmelCase=13						,	_UpperCAmelCase=7						,	_UpperCAmelCase=True						,	_UpperCAmelCase=True						,	_UpperCAmelCase=True						,	_UpperCAmelCase=True						,	_UpperCAmelCase=99						,	_UpperCAmelCase=32						,	_UpperCAmelCase=2						,	_UpperCAmelCase=4						,	_UpperCAmelCase=37						,	_UpperCAmelCase="gelu"						,	_UpperCAmelCase=0.1						,	_UpperCAmelCase=0.1						,	_UpperCAmelCase=512						,	_UpperCAmelCase=16						,	_UpperCAmelCase=2						,	_UpperCAmelCase=0.02						,	_UpperCAmelCase=3						,	_UpperCAmelCase=4						,	_UpperCAmelCase=None						,	_UpperCAmelCase=1000						,	):

          '''simple docstring'''






          __A						: List[str]      =       parent
          __A						: List[str]      =       batch_size
          __A						: Tuple      =       seq_length
          __A						: Tuple      =       is_training
          __A						: Dict      =       use_input_mask
          __A						: Any      =       use_token_type_ids
          __A						: Optional[Any]      =       use_labels
          __A						: Optional[int]      =       vocab_size
          __A						: int      =       hidden_size
          __A						: str      =       num_hidden_layers
          __A						: str      =       num_attention_heads
          __A						: List[Any]      =       intermediate_size
          __A						: Union[str, Any]      =       hidden_act
          __A						: Dict      =       hidden_dropout_prob
          __A						: Any      =       attention_probs_dropout_prob
          __A						: int      =       max_position_embeddings
          __A						: int      =       type_vocab_size
          __A						: Optional[int]      =       type_sequence_label_size
          __A						: Tuple      =       initializer_range
          __A						: Any      =       num_labels
          __A						: List[str]      =       num_choices
          __A						: List[str]      =       scope
          __A						: Any      =       range_bbox


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: str      =       ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size)

          # convert bbox to numpy since TF does not support item assignment
          __A						: Optional[int]      =       ids_tensor([self.batch_size, self.seq_length, 4]						,	self.range_bbox).numpy()
          # Ensure that bbox is legal
          for i in range(bbox.shape[0]):
               for j in range(bbox.shape[1]):
                    if bbox[i, j, 3] < bbox[i, j, 1]:
                         __A						: Union[str, Any]      =       bbox[i, j, 3]
                         __A						: str      =       bbox[i, j, 1]
                         __A						: Optional[int]      =       t
                    if bbox[i, j, 2] < bbox[i, j, 0]:
                         __A						: Optional[Any]      =       bbox[i, j, 2]
                         __A						: Optional[Any]      =       bbox[i, j, 0]
                         __A						: str      =       t
          __A						: List[str]      =       tf.convert_to_tensor(_UpperCAmelCase)

          __A						: Tuple      =       None
          if self.use_input_mask:
               __A						: Tuple      =       random_attention_mask([self.batch_size, self.seq_length])

          __A						: Any      =       None
          if self.use_token_type_ids:
               __A						: List[Any]      =       ids_tensor([self.batch_size, self.seq_length]						,	self.type_vocab_size)

          __A						: int      =       None
          __A						: Optional[Any]      =       None
          __A						: Any      =       None
          if self.use_labels:
               __A						: List[Any]      =       ids_tensor([self.batch_size]						,	self.type_sequence_label_size)
               __A						: Any      =       ids_tensor([self.batch_size, self.seq_length]						,	self.num_labels)
               __A						: Union[str, Any]      =       ids_tensor([self.batch_size]						,	self.num_choices)

          __A						: List[str]      =       LayoutLMConfig(
              vocab_size=self.vocab_size						,	hidden_size=self.hidden_size						,	num_hidden_layers=self.num_hidden_layers						,	num_attention_heads=self.num_attention_heads						,	intermediate_size=self.intermediate_size						,	hidden_act=self.hidden_act						,	hidden_dropout_prob=self.hidden_dropout_prob						,	attention_probs_dropout_prob=self.attention_probs_dropout_prob						,	max_position_embeddings=self.max_position_embeddings						,	type_vocab_size=self.type_vocab_size						,	initializer_range=self.initializer_range						,	)

          return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase):

          '''simple docstring'''






          __A						: Tuple      =       TFLayoutLMModel(config=_UpperCAmelCase)

          __A						: List[Any]      =       model(_UpperCAmelCase						,	_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase)
          __A						: Tuple      =       model(_UpperCAmelCase						,	_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase)
          __A						: str      =       model(_UpperCAmelCase						,	_UpperCAmelCase)
          self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size))
          self.parent.assertEqual(result.pooler_output.shape						,	(self.batch_size, self.hidden_size))


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase):

          '''simple docstring'''






          __A						: Optional[Any]      =       TFLayoutLMForMaskedLM(config=_UpperCAmelCase)

          __A						: Any      =       model(_UpperCAmelCase						,	_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase						,	labels=_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.vocab_size))


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase):

          '''simple docstring'''






          __A						: int      =       self.num_labels
          __A						: str      =       TFLayoutLMForSequenceClassification(config=_UpperCAmelCase)

          __A						: Optional[int]      =       model(_UpperCAmelCase						,	_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.num_labels))


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase):

          '''simple docstring'''






          __A						: str      =       self.num_labels
          __A						: Optional[Any]      =       TFLayoutLMForTokenClassification(config=_UpperCAmelCase)

          __A						: Optional[Any]      =       model(_UpperCAmelCase						,	_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase						,	labels=_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.num_labels))


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase):

          '''simple docstring'''






          __A						: int      =       TFLayoutLMForQuestionAnswering(config=_UpperCAmelCase)

          __A						: Optional[int]      =       model(_UpperCAmelCase						,	_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase)
          self.parent.assertEqual(result.start_logits.shape						,	(self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape						,	(self.batch_size, self.seq_length))


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Any      =       self.prepare_config_and_inputs()
          (
              (
              __A
          ) ,(
              __A
          ) ,(
              __A
          ) ,(
              __A
          ) ,(
              __A
          ) ,(
              __A
          ) ,(
              __A
          ) ,(
              __A
          ) ,
          )						: Optional[int]      =       config_and_inputs
          __A						: Optional[int]      =       {
              'input_ids': input_ids,
              'bbox': bbox,
              'token_type_ids': token_type_ids,
              'attention_mask': input_mask,
          }
          return config, inputs_dict




@require_tf
class  SCREAMING_SNAKE_CASE    (a__					,		a__					,		unittest.TestCase							):
     lowerCAmelCase					=       (
         (
             TFLayoutLMModel,
             TFLayoutLMForMaskedLM,
             TFLayoutLMForTokenClassification,
             TFLayoutLMForSequenceClassification,
             TFLayoutLMForQuestionAnswering,
         )
         if is_tf_available()
         else ()
     )
     lowerCAmelCase					=       (
         {
             '''feature-extraction''': TFLayoutLMModel,
             '''fill-mask''': TFLayoutLMForMaskedLM,
             '''text-classification''': TFLayoutLMForSequenceClassification,
             '''token-classification''': TFLayoutLMForTokenClassification,
             '''zero-shot''': TFLayoutLMForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     lowerCAmelCase					=       False
     lowerCAmelCase					=       True
     lowerCAmelCase					=       10


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[str]      =       TFLayoutLMModelTester(self)
          __A						: Any      =       ConfigTester(self						,	config_class=_UpperCAmelCase						,	hidden_size=37)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.config_tester.run_common_tests()


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: str      =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[str]      =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[str]      =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: str      =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[Any]      =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase)


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               __A						: Union[str, Any]      =       TFLayoutLMModel.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)


     @unittest.skip('Onnx compliancy broke with TF 2.10')
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          pass






def        _lowerCAmelCase			(  )     ->     List[Any]:
     # Here we prepare a batch of 2 sequences to test a LayoutLM forward pass on:
     # fmt: off
     __A						: Any      =       tf.convert_to_tensor([[1_01,10_19,10_14,10_16,10_37,1_28_49,47_47,10_04,1_42_46,22_78,54_39,45_24,50_02,29_30,21_93,29_30,43_41,32_08,10_05,10_55,21_71,28_48,1_13_00,35_31,1_02],[1_01,40_70,40_34,70_20,10_24,30_58,10_15,10_13,28_61,10_13,60_70,1_92_74,27_72,62_05,2_78_14,1_61_47,1_61_47,43_43,20_47,1_02_83,1_09_69,1_43_89,10_12,23_38,1_02]]			)  # noqa: E231
     __A						: Any      =       tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]			)  # noqa: E231
     __A						: Optional[int]      =       tf.convert_to_tensor([[[0,0,0,0],[4_23,2_37,4_40,2_51],[4_27,2_72,4_41,2_87],[4_19,1_15,4_37,1_29],[9_61,8_85,9_92,9_12],[2_56,38,3_30,58],[2_56,38,3_30,58],[3_36,42,3_53,57],[3_60,39,4_01,56],[3_60,39,4_01,56],[4_11,39,4_71,59],[4_79,41,5_28,59],[5_33,39,6_30,60],[67,1_13,1_34,1_31],[1_41,1_15,2_09,1_32],[68,1_49,1_33,1_66],[1_41,1_49,1_87,1_64],[1_95,1_48,2_87,1_65],[1_95,1_48,2_87,1_65],[1_95,1_48,2_87,1_65],[2_95,1_48,3_49,1_65],[4_41,1_49,4_92,1_66],[4_97,1_49,5_46,1_64],[64,2_01,1_25,2_18],[10_00,10_00,10_00,10_00]],[[0,0,0,0],[6_62,1_50,7_54,1_66],[6_65,1_99,7_42,2_11],[5_19,2_13,5_54,2_28],[5_19,2_13,5_54,2_28],[1_34,4_33,1_87,4_54],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[3_14,4_69,3_76,4_82],[5_04,6_84,5_82,7_06],[9_41,8_25,9_73,9_00],[9_41,8_25,9_73,9_00],[9_41,8_25,9_73,9_00],[9_41,8_25,9_73,9_00],[6_10,7_49,6_52,7_65],[1_30,6_59,1_68,6_72],[1_76,6_57,2_37,6_72],[2_38,6_57,3_12,6_72],[4_43,6_53,6_28,6_72],[4_43,6_53,6_28,6_72],[7_16,3_01,8_25,3_17],[10_00,10_00,10_00,10_00]]]			)  # noqa: E231
     __A						: int      =       tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]			)  # noqa: E231
     # these are sequence labels (i.e. at the token level)
     __A						: Dict      =       tf.convert_to_tensor([[-1_00,10,10,10,9,1,-1_00,7,7,-1_00,7,7,4,2,5,2,8,8,-1_00,-1_00,5,0,3,2,-1_00],[-1_00,12,12,12,-1_00,12,10,-1_00,-1_00,-1_00,-1_00,10,12,9,-1_00,-1_00,-1_00,10,10,10,9,12,-1_00,10,-1_00]]			)  # noqa: E231
     # fmt: on

     return input_ids, attention_mask, bbox, token_type_ids, labels







@require_tf
class  SCREAMING_SNAKE_CASE    (unittest.TestCase							):


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[Any]      =       TFLayoutLMModel.from_pretrained('microsoft/layoutlm-base-uncased')

          __A ,__A ,__A ,__A ,__A						: List[str]      =       prepare_layoutlm_batch_inputs()

          # forward pass
          __A						: List[str]      =       model(input_ids=_UpperCAmelCase						,	bbox=_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase)

          # test the sequence output on [0, :3, :3]
          __A						: Any      =       tf.convert_to_tensor(
              [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]]						,	)

          self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]						,	_UpperCAmelCase						,	atol=1e-3))

          # test the pooled output on [1, :3]
          __A						: str      =       tf.convert_to_tensor([-0.6580, -0.0214, 0.8552])

          self.assertTrue(np.allclose(outputs.pooler_output[1, :3]						,	_UpperCAmelCase						,	atol=1e-3))


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Tuple      =       TFLayoutLMForSequenceClassification.from_pretrained('microsoft/layoutlm-base-uncased'						,	num_labels=2)

          __A ,__A ,__A ,__A ,__A						: Union[str, Any]      =       prepare_layoutlm_batch_inputs()

          # forward pass
          __A						: int      =       model(
              input_ids=_UpperCAmelCase						,	bbox=_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase						,	labels=tf.convert_to_tensor([1, 1])						,	)

          # test whether we get a loss as a scalar
          __A						: Any      =       outputs.loss
          __A						: Union[str, Any]      =       (2,)
          self.assertEqual(loss.shape						,	_UpperCAmelCase)

          # test the shape of the logits
          __A						: int      =       outputs.logits
          __A						: Optional[int]      =       (2, 2)
          self.assertEqual(logits.shape						,	_UpperCAmelCase)


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[int]      =       TFLayoutLMForTokenClassification.from_pretrained('microsoft/layoutlm-base-uncased'						,	num_labels=13)

          __A ,__A ,__A ,__A ,__A						: Any      =       prepare_layoutlm_batch_inputs()

          # forward pass
          __A						: Optional[Any]      =       model(
              input_ids=_UpperCAmelCase						,	bbox=_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase						,	labels=_UpperCAmelCase)

          # test the shape of the logits
          __A						: Optional[Any]      =       outputs.logits
          __A						: List[Any]      =       tf.convert_to_tensor((2, 25, 13))
          self.assertEqual(logits.shape						,	_UpperCAmelCase)


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       TFLayoutLMForQuestionAnswering.from_pretrained('microsoft/layoutlm-base-uncased')

          __A ,__A ,__A ,__A ,__A						: Union[str, Any]      =       prepare_layoutlm_batch_inputs()

          # forward pass
          __A						: List[Any]      =       model(input_ids=_UpperCAmelCase						,	bbox=_UpperCAmelCase						,	attention_mask=_UpperCAmelCase						,	token_type_ids=_UpperCAmelCase)

          # test the shape of the logits
          __A						: Union[str, Any]      =       tf.convert_to_tensor((2, 25))
          self.assertEqual(outputs.start_logits.shape						,	_UpperCAmelCase)
          self.assertEqual(outputs.end_logits.shape						,	_UpperCAmelCase)






'''simple docstring'''




from argparse import ArgumentParser

from datasets.commands.convert import ConvertCommand
from datasets.commands.dummy_data import DummyDataCommand
from datasets.commands.env import EnvironmentCommand
from datasets.commands.run_beam import RunBeamCommand
from datasets.commands.test import TestCommand
from datasets.utils.logging import set_verbosity_info






def        _lowerCAmelCase			(  __snake_case  : Tuple			)     ->     Dict:
     return {key.lstrip('-'			): value for key, value in zip(unknown_args[::2] ,							unknown_args[1::2]			)}






def        _lowerCAmelCase			(  )     ->     Tuple:
     __A						: int      =       ArgumentParser(
         'HuggingFace Datasets CLI tool' ,							usage='datasets-cli <command> [<args>]' ,							allow_abbrev=__snake_case			)
     __A						: Optional[Any]      =       parser.add_subparsers(help='datasets-cli command helpers'			)
     set_verbosity_info()

     # Register commands
     ConvertCommand.register_subcommand(__snake_case			)
     EnvironmentCommand.register_subcommand(__snake_case			)
     TestCommand.register_subcommand(__snake_case			)
     RunBeamCommand.register_subcommand(__snake_case			)
     DummyDataCommand.register_subcommand(__snake_case			)

     # Parse args
     __A ,__A						: Optional[Any]      =       parser.parse_known_args()
     if not hasattr(__snake_case ,							'func'			):
          parser.print_help()
          exit(1			)
     __A						: Any      =       parse_unknown_args(__snake_case			)

     # Run
     __A						: List[Any]      =       args.func(__snake_case ,							**__snake_case			)
     service.run()


if __name__ == "__main__":
  main()