
'''simple docstring'''





import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


a__		: Union[str, Any]  							=							logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

a__		: List[str]  							=							{
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
a__		: List[str]  							=							{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
a__		: Optional[Any]  							=							sorted(arg_to_scheduler.keys())
a__		: Tuple  							=							"{" + ", ".join(arg_to_scheduler_choices) + "}"


class 			UpperCamelCase__	(     pl.LightningModule):






 def __init__(      self	:Optional[Any]						,					_A	:argparse.Namespace						,					_A	:Optional[int]=None						,					_A	:List[Any]="base"						,					_A	:Tuple=None						,					_A	:List[Any]=None						,					_A	:Optional[Any]=None						,					**_A	:Optional[int]						,					)					->      Optional[Any]:







       '''simple docstring'''



       super().__init__()
       # TODO: move to self.save_hyperparameters()
       # self.save_hyperparameters()
       # can also expand arguments into trainer signature for easier reading

       self.save_hyperparameters(_A      )
       __A													=      0
       __A													=      Path(self.hparams.output_dir      )
       __A													=      self.hparams.cache_dir if self.hparams.cache_dir else None
       if config is None:
             __A													=      AutoConfig.from_pretrained(
                 self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path						,					**({'num_labels': num_labels} if num_labels is not None else {})						,					cache_dir=_A						,					**_A						,					)
       else:
             __A													=      config

       __A													=      ('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
       for p in extra_model_params:
             if getattr(self.hparams						,					_A						,					_A      ):
                   assert hasattr(self.config						,					_A      ), F'model config doesn\'t have a `{p}` attribute'
                   setattr(self.config						,					_A						,					getattr(self.hparams						,					_A      )      )

       if tokenizer is None:
             __A													=      AutoTokenizer.from_pretrained(
                 self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path						,					cache_dir=_A						,					)
       else:
             __A													=      tokenizer
       __A													=      MODEL_MODES[mode]
       if model is None:
             __A													=      self.model_type.from_pretrained(
                 self.hparams.model_name_or_path						,					from_tf=bool('.ckpt' in self.hparams.model_name_or_path      )						,					config=self.config						,					cache_dir=_A						,					)
       else:
             __A													=      model






 def 						lowercase_    (      self	:str						,					*_A	:str						,					**_A	:Dict      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      self.model_type.from_pretrained(*_A						,					**_A      )






 def 						lowercase_    (      self	:Tuple      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      arg_to_scheduler[self.hparams.lr_scheduler]
       __A													=      get_schedule_func(
           self.opt						,					num_warmup_steps=self.hparams.warmup_steps						,					num_training_steps=self.total_steps()      )
       __A													=      {'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
       return scheduler






 def 						lowercase_    (      self	:List[str]      )					->      Tuple:







       '''simple docstring'''



       __A													=      self.model
       __A													=      ['bias', 'LayerNorm.weight']
       __A													=      [
           {
               'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay      )],
               'weight_decay': self.hparams.weight_decay,
           },
           {
               'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay      )],
               'weight_decay': 0.0,
           },
       ]
       if self.hparams.adafactor:
             __A													=      Adafactor(
                 _A						,					lr=self.hparams.learning_rate						,					scale_parameter=_A						,					relative_step=_A      )

       else:
             __A													=      AdamW(
                 _A						,					lr=self.hparams.learning_rate						,					eps=self.hparams.adam_epsilon      )
       __A													=      optimizer

       __A													=      self.get_lr_scheduler()

       return [optimizer], [scheduler]






 def 						lowercase_    (      self	:Tuple						,					_A	:Tuple						,					_A	:str      )					->      Union[str, Any]:







       '''simple docstring'''



       return self.validation_step(_A						,					_A      )






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:Optional[int]      )					->      List[Any]:







       '''simple docstring'''



       return self.validation_end(_A      )






 def 						lowercase_    (      self	:Optional[int]      )					->      int:







       '''simple docstring'''



       __A													=      max(1						,					self.hparams.gpus      )  # TODO: consider num_tpu_cores
       __A													=      self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
       return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs






 def 						lowercase_    (      self	:int						,					_A	:Optional[int]      )					->      Optional[int]:







       '''simple docstring'''



       if mode == "test":
             __A													=      len(self.test_dataloader().dataset      )
       else:
             __A													=      self.get_dataloader('train'						,					self.hparams.train_batch_size						,					shuffle=_A      )
             __A													=      len(self.train_dataloader().dataset      )






 def 						lowercase_    (      self	:int						,					_A	:str						,					_A	:int						,					_A	:bool = False      )					->      Union[str, Any]:







       '''simple docstring'''



       raise NotImplementedError('You must implement this for your task'      )






 def 						lowercase_    (      self	:Optional[Any]      )					->      str:







       '''simple docstring'''



       return self.train_loader






 def 						lowercase_    (      self	:Optional[int]      )					->      Union[str, Any]:







       '''simple docstring'''



       return self.get_dataloader('dev'						,					self.hparams.eval_batch_size						,					shuffle=_A      )






 def 						lowercase_    (      self	:List[str]      )					->      Union[str, Any]:







       '''simple docstring'''



       return self.get_dataloader('test'						,					self.hparams.eval_batch_size						,					shuffle=_A      )






 def 						lowercase_    (      self	:str						,					_A	:Union[str, Any]      )					->      List[Any]:







       '''simple docstring'''



       return os.path.join(
           self.hparams.data_dir						,					'cached_{}_{}_{}'.format(
               _A						,					list(filter(_A						,					self.hparams.model_name_or_path.split('/'      )      )      ).pop()						,					str(self.hparams.max_seq_length      )						,					)						,					)






 @pl.utilities.rank_zero_only
 def 						lowercase_    (      self	:List[str]						,					_A	:Dict[str, Any]      )					->      None:







       '''simple docstring'''



       __A													=      self.output_dir.joinpath('best_tfmr'      )
       __A													=      self.step_count
       self.model.save_pretrained(_A      )
       self.tokenizer.save_pretrained(_A      )






 @staticmethod
 def 						lowercase_    (      _A	:Dict						,					_A	:Union[str, Any]      )					->      Optional[int]:







       '''simple docstring'''



       parser.add_argument(
           '--model_name_or_path'						,					default=_A						,					type=_A						,					required=_A						,					help='Path to pretrained model or model identifier from huggingface.co/models'						,					)
       parser.add_argument(
           '--config_name'						,					default=''						,					type=_A						,					help='Pretrained config name or path if not the same as model_name'      )
       parser.add_argument(
           '--tokenizer_name'						,					default=_A						,					type=_A						,					help='Pretrained tokenizer name or path if not the same as model_name'						,					)
       parser.add_argument(
           '--cache_dir'						,					default=''						,					type=_A						,					help='Where do you want to store the pre-trained models downloaded from huggingface.co'						,					)
       parser.add_argument(
           '--encoder_layerdrop'						,					type=_A						,					help='Encoder layer dropout probability (Optional). Goes into model.config'						,					)
       parser.add_argument(
           '--decoder_layerdrop'						,					type=_A						,					help='Decoder layer dropout probability (Optional). Goes into model.config'						,					)
       parser.add_argument(
           '--dropout'						,					type=_A						,					help='Dropout probability (Optional). Goes into model.config'						,					)
       parser.add_argument(
           '--attention_dropout'						,					type=_A						,					help='Attention dropout probability (Optional). Goes into model.config'						,					)
       parser.add_argument('--learning_rate'						,					default=5E-5						,					type=_A						,					help='The initial learning rate for Adam.'      )
       parser.add_argument(
           '--lr_scheduler'						,					default='linear'						,					choices=_A						,					metavar=_A						,					type=_A						,					help='Learning rate scheduler'						,					)
       parser.add_argument('--weight_decay'						,					default=0.0						,					type=_A						,					help='Weight decay if we apply some.'      )
       parser.add_argument('--adam_epsilon'						,					default=1E-8						,					type=_A						,					help='Epsilon for Adam optimizer.'      )
       parser.add_argument('--warmup_steps'						,					default=0						,					type=_A						,					help='Linear warmup over warmup_steps.'      )
       parser.add_argument('--num_workers'						,					default=4						,					type=_A						,					help='kwarg passed to DataLoader'      )
       parser.add_argument('--num_train_epochs'						,					dest='max_epochs'						,					default=3						,					type=_A      )
       parser.add_argument('--train_batch_size'						,					default=32						,					type=_A      )
       parser.add_argument('--eval_batch_size'						,					default=32						,					type=_A      )
       parser.add_argument('--adafactor'						,					action='store_true'      )


class 			UpperCamelCase__	(     pl.Callback):






 def 						lowercase_    (      self	:Tuple						,					_A	:Tuple						,					_A	:Dict      )					->      int:







       '''simple docstring'''



       __A													=      trainer.lr_schedulers[0]['scheduler']
       __A													=      {F'lr_group_{i}': lr for i, lr in enumerate(lr_scheduler.get_lr()      )}
       pl_module.logger.log_metrics(_A      )






 def 						lowercase_    (      self	:Dict						,					_A	:pl.Trainer						,					_A	:pl.LightningModule      )					->      str:







       '''simple docstring'''



       rank_zero_info('***** Validation results *****'      )
       __A													=      trainer.callback_metrics
       # Log results
       for key in sorted(_A      ):
             if key not in ["log", "progress_bar"]:
                   rank_zero_info('{} = {}\n'.format(_A						,					str(metrics[key]      )      )      )






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:pl.Trainer						,					_A	:pl.LightningModule      )					->      str:







       '''simple docstring'''



       rank_zero_info('***** Test results *****'      )
       __A													=      trainer.callback_metrics
       # Log and save results to file
       __A													=      os.path.join(pl_module.hparams.output_dir						,					'test_results.txt'      )
       with open(_A						,					'w'      ) as writer:
             for key in sorted(_A      ):
                   if key not in ["log", "progress_bar"]:
                         rank_zero_info('{} = {}\n'.format(_A						,					str(metrics[key]      )      )      )
                         writer.write('{} = {}\n'.format(_A						,					str(metrics[key]      )      )      )





def 							snake_case				(		UpperCAmelCase   ,	UpperCAmelCase						)->      None:







      """simple docstring"""
      #  To allow all pl args uncomment the following line
      #  parser = pl.Trainer.add_argparse_args(parser)
      parser.add_argument(
          '--output_dir'   ,	default=UpperCAmelCase   ,	type=UpperCAmelCase   ,	required=UpperCAmelCase   ,	help='The output directory where the model predictions and checkpoints will be written.'   ,	)
      parser.add_argument(
          '--fp16'   ,	action='store_true'   ,	help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'   ,	)

      parser.add_argument(
          '--fp16_opt_level'   ,	type=UpperCAmelCase   ,	default='O2'   ,	help=(
              'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
              'See details at https://nvidia.github.io/apex/amp.html'
          )   ,	)
      parser.add_argument('--n_tpu_cores'   ,	dest='tpu_cores'   ,	type=UpperCAmelCase						)
      parser.add_argument('--max_grad_norm'   ,	dest='gradient_clip_val'   ,	default=1.0   ,	type=UpperCAmelCase   ,	help='Max gradient norm'						)
      parser.add_argument('--do_train'   ,	action='store_true'   ,	help='Whether to run training.'						)
      parser.add_argument('--do_predict'   ,	action='store_true'   ,	help='Whether to run predictions on the test set.'						)
      parser.add_argument(
          '--gradient_accumulation_steps'   ,	dest='accumulate_grad_batches'   ,	type=UpperCAmelCase   ,	default=1   ,	help='Number of updates steps to accumulate before performing a backward/update pass.'   ,	)
      parser.add_argument('--seed'   ,	type=UpperCAmelCase   ,	default=4_2   ,	help='random seed for initialization'						)
      parser.add_argument(
          '--data_dir'   ,	default=UpperCAmelCase   ,	type=UpperCAmelCase   ,	required=UpperCAmelCase   ,	help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'   ,	)





def 							snake_case				(		UpperCAmelCase   ,	UpperCAmelCase   ,	UpperCAmelCase=None   ,	UpperCAmelCase=True   ,	UpperCAmelCase=[]   ,	UpperCAmelCase=None   ,	UpperCAmelCase=None   ,	**UpperCAmelCase   ,	)->      Optional[Any]:







      """simple docstring"""
      pl.seed_everything(args.seed						)

      # init model
      __A													=      Path(model.hparams.output_dir						)
      odir.mkdir(exist_ok=UpperCAmelCase						)

      # add custom checkpoints
      if checkpoint_callback is None:
            __A													=      pl.callbacks.ModelCheckpoint(
                filepath=args.output_dir   ,	prefix='checkpoint'   ,	monitor='val_loss'   ,	mode='min'   ,	save_top_k=1						)
      if early_stopping_callback:
            extra_callbacks.append(UpperCAmelCase						)
      if logging_callback is None:
            __A													=      LoggingCallback()

      __A													=      {}

      # TODO: remove with PyTorch 1.6 since pl uses native amp
      if args.fpaa:
            __A													=      1_6
            __A													=      args.fpaa_opt_level

      if args.gpus > 1:
            __A													=      'ddp'

      __A													=      args.accumulate_grad_batches
      __A													=      extra_train_kwargs.get('accelerator'   ,	UpperCAmelCase						)
      __A													=      extra_train_kwargs.get('profiler'   ,	UpperCAmelCase						)

      __A													=      pl.Trainer.from_argparse_args(
          UpperCAmelCase   ,	weights_summary=UpperCAmelCase   ,	callbacks=[logging_callback] + extra_callbacks   ,	logger=UpperCAmelCase   ,	checkpoint_callback=UpperCAmelCase   ,	**UpperCAmelCase   ,	)

      if args.do_train:
            trainer.fit(UpperCAmelCase						)

      return trainer








'''simple docstring'''





import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MarkupLMForQuestionAnswering,
     MarkupLMForSequenceClassification,
     MarkupLMForTokenClassification,
     MarkupLMModel,
 )

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer


class 			UpperCamelCase__	:






 def __init__(      self	:str						,					_A	:Any						,					_A	:Union[str, Any]=13						,					_A	:List[Any]=7						,					_A	:int=True						,					_A	:Union[str, Any]=True						,					_A	:List[Any]=True						,					_A	:Dict=True						,					_A	:int=99						,					_A	:Union[str, Any]=32						,					_A	:int=5						,					_A	:List[str]=4						,					_A	:Optional[int]=37						,					_A	:List[str]="gelu"						,					_A	:Any=0.1						,					_A	:int=0.1						,					_A	:List[str]=512						,					_A	:Optional[Any]=16						,					_A	:Optional[Any]=2						,					_A	:List[str]=0.02						,					_A	:Any=3						,					_A	:Tuple=None						,					_A	:str=20						,					_A	:List[Any]=30						,					_A	:Dict=2						,					_A	:Union[str, Any]=2						,					_A	:int=10						,					)					->      Any:







       '''simple docstring'''



       __A													=      parent
       __A													=      batch_size
       __A													=      seq_length
       __A													=      is_training
       __A													=      use_input_mask
       __A													=      use_token_type_ids
       __A													=      use_labels
       __A													=      vocab_size
       __A													=      hidden_size
       __A													=      num_hidden_layers
       __A													=      num_attention_heads
       __A													=      intermediate_size
       __A													=      hidden_act
       __A													=      hidden_dropout_prob
       __A													=      attention_probs_dropout_prob
       __A													=      max_position_embeddings
       __A													=      type_vocab_size
       __A													=      type_sequence_label_size
       __A													=      initializer_range
       __A													=      num_labels
       __A													=      scope
       __A													=      max_xpath_tag_unit_embeddings
       __A													=      max_xpath_subs_unit_embeddings
       __A													=      tag_pad_id
       __A													=      subs_pad_id
       __A													=      max_depth






 def 						lowercase_    (      self	:Tuple      )					->      Optional[int]:







       '''simple docstring'''



       __A													=      ids_tensor([self.batch_size, self.seq_length]						,					self.vocab_size      )

       __A													=      ids_tensor(
           [self.batch_size, self.seq_length, self.max_depth]						,					self.max_xpath_tag_unit_embeddings      )

       __A													=      ids_tensor(
           [self.batch_size, self.seq_length, self.max_depth]						,					self.max_xpath_subs_unit_embeddings      )

       __A													=      None
       if self.use_input_mask:
             __A													=      ids_tensor([self.batch_size, self.seq_length]						,					vocab_size=2      )

       __A													=      None
       if self.use_token_type_ids:
             __A													=      ids_tensor([self.batch_size, self.seq_length]						,					self.type_vocab_size      )

       __A													=      None
       __A													=      None
       if self.use_labels:
             __A													=      ids_tensor([self.batch_size]						,					self.type_sequence_label_size      )
             __A													=      ids_tensor([self.batch_size, self.seq_length]						,					self.num_labels      )

       __A													=      self.get_config()

       return (
           config,
           input_ids,
           xpath_tags_seq,
           xpath_subs_seq,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
       )






 def 						lowercase_    (      self	:List[Any]      )					->      int:







       '''simple docstring'''



       return MarkupLMConfig(
           vocab_size=self.vocab_size						,					hidden_size=self.hidden_size						,					num_hidden_layers=self.num_hidden_layers						,					num_attention_heads=self.num_attention_heads						,					intermediate_size=self.intermediate_size						,					hidden_act=self.hidden_act						,					hidden_dropout_prob=self.hidden_dropout_prob						,					attention_probs_dropout_prob=self.attention_probs_dropout_prob						,					max_position_embeddings=self.max_position_embeddings						,					type_vocab_size=self.type_vocab_size						,					initializer_range=self.initializer_range						,					max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings						,					max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings						,					tag_pad_id=self.tag_pad_id						,					subs_pad_id=self.subs_pad_id						,					max_depth=self.max_depth						,					)






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:List[str]						,					_A	:str						,					_A	:int						,					_A	:str						,					_A	:Dict						,					_A	:str						,					_A	:Optional[int]						,					_A	:Optional[Any]						,					)					->      Tuple:







       '''simple docstring'''



       __A													=      MarkupLMModel(config=_A      )
       model.to(_A      )
       model.eval()
       print('Configs:'						,					model.config.tag_pad_id						,					model.config.subs_pad_id      )
       __A													=      model(_A						,					attention_mask=_A						,					token_type_ids=_A      )
       __A													=      model(_A						,					token_type_ids=_A      )
       __A													=      model(_A      )
       self.parent.assertEqual(result.last_hidden_state.shape						,					(self.batch_size, self.seq_length, self.hidden_size)      )
       self.parent.assertEqual(result.pooler_output.shape						,					(self.batch_size, self.hidden_size)      )






 def 						lowercase_    (      self	:Any						,					_A	:Optional[Any]						,					_A	:Any						,					_A	:Optional[int]						,					_A	:Any						,					_A	:int						,					_A	:List[Any]						,					_A	:Dict						,					_A	:List[str]						,					)					->      Any:







       '''simple docstring'''



       __A													=      self.num_labels
       __A													=      MarkupLMForSequenceClassification(_A      )
       model.to(_A      )
       model.eval()
       __A													=      model(
           _A						,					xpath_tags_seq=_A						,					xpath_subs_seq=_A						,					attention_mask=_A						,					token_type_ids=_A						,					labels=_A						,					)
       self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.num_labels)      )






 def 						lowercase_    (      self	:Any						,					_A	:List[str]						,					_A	:str						,					_A	:str						,					_A	:List[Any]						,					_A	:Optional[Any]						,					_A	:Union[str, Any]						,					_A	:Dict						,					_A	:Union[str, Any]						,					)					->      Dict:







       '''simple docstring'''



       __A													=      self.num_labels
       __A													=      MarkupLMForTokenClassification(config=_A      )
       model.to(_A      )
       model.eval()
       __A													=      model(
           _A						,					xpath_tags_seq=_A						,					xpath_subs_seq=_A						,					attention_mask=_A						,					token_type_ids=_A						,					labels=_A						,					)
       self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.seq_length, self.num_labels)      )






 def 						lowercase_    (      self	:int						,					_A	:str						,					_A	:Tuple						,					_A	:Any						,					_A	:int						,					_A	:Any						,					_A	:Any						,					_A	:Union[str, Any]						,					_A	:List[Any]						,					)					->      List[str]:







       '''simple docstring'''



       __A													=      MarkupLMForQuestionAnswering(config=_A      )
       model.to(_A      )
       model.eval()
       __A													=      model(
           _A						,					xpath_tags_seq=_A						,					xpath_subs_seq=_A						,					attention_mask=_A						,					token_type_ids=_A						,					start_positions=_A						,					end_positions=_A						,					)
       self.parent.assertEqual(result.start_logits.shape						,					(self.batch_size, self.seq_length)      )
       self.parent.assertEqual(result.end_logits.shape						,					(self.batch_size, self.seq_length)      )






 def 						lowercase_    (      self	:Dict      )					->      List[str]:







       '''simple docstring'''



       __A													=      self.prepare_config_and_inputs()
       (
           (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   
       )													=      config_and_inputs
       __A													=      {
           'input_ids': input_ids,
           'xpath_tags_seq': xpath_tags_seq,
           'xpath_subs_seq': xpath_subs_seq,
           'token_type_ids': token_type_ids,
           'attention_mask': input_mask,
       }
       return config, inputs_dict


@require_torch
class 			UpperCamelCase__	(     SCREAMING_SNAKE_CASE   ,  SCREAMING_SNAKE_CASE   ,  unittest.TestCase):
 UpperCAmelCase__     :     List[Any]    					=       (
     (
         MarkupLMModel,
         MarkupLMForSequenceClassification,
         MarkupLMForTokenClassification,
         MarkupLMForQuestionAnswering,
     )
     if is_torch_available()
     else None
 )
 UpperCAmelCase__     :     Tuple    					=       (
     {
         'feature-extraction': MarkupLMModel,
         'question-answering': MarkupLMForQuestionAnswering,
         'text-classification': MarkupLMForSequenceClassification,
         'token-classification': MarkupLMForTokenClassification,
         'zero-shot': MarkupLMForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:List[str]						,					_A	:Optional[Any]						,					_A	:Dict						,					_A	:Optional[Any]						,					_A	:Optional[int]      )					->      Tuple:







       '''simple docstring'''



       return True






 def 						lowercase_    (      self	:List[str]      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      MarkupLMModelTester(self      )
       __A													=      ConfigTester(self						,					config_class=_A						,					hidden_size=37      )






 def 						lowercase_    (      self	:Optional[Any]      )					->      int:







       '''simple docstring'''



       self.config_tester.run_common_tests()






 def 						lowercase_    (      self	:Union[str, Any]      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_A      )






 def 						lowercase_    (      self	:Optional[Any]      )					->      Any:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_A      )






 def 						lowercase_    (      self	:Tuple      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_A      )






 def 						lowercase_    (      self	:List[Any]      )					->      Tuple:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_A      )





def 							snake_case				(		)->      str:







      """simple docstring"""
      __A													=      '\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    '

      return html_string


@require_torch
class 			UpperCamelCase__	(     unittest.TestCase):






 @cached_property
 def 						lowercase_    (      self	:Optional[int]      )					->      Dict:







       '''simple docstring'''



       __A													=      MarkupLMFeatureExtractor()
       __A													=      MarkupLMTokenizer.from_pretrained('microsoft/markuplm-base'      )

       return MarkupLMProcessor(_A						,					_A      )






 @slow
 def 						lowercase_    (      self	:Any      )					->      str:







       '''simple docstring'''



       __A													=      MarkupLMModel.from_pretrained('microsoft/markuplm-base'      ).to(_A      )

       __A													=      self.default_processor

       __A													=      processor(prepare_html_string()						,					return_tensors='pt'      )
       __A													=      inputs.to(_A      )

       # forward pass
       with torch.no_grad():
             __A													=      model(**_A      )

       # verify the last hidden states
       __A													=      torch.Size([1, 14, 768]      )
       self.assertEqual(outputs.last_hidden_state.shape						,					_A      )

       __A													=      torch.tensor(
           [[0.0_675, -0.0_052, 0.5_001], [-0.2_281, 0.0_802, 0.2_192], [-0.0_583, -0.3_311, 0.1_185]]      ).to(_A      )

       self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]						,					_A						,					atol=1E-4      )      )






