



import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCAmelCase :				Tuple							=16
UpperCAmelCase :				str							=32



def  _lowerCAmelCase     (_lowerCAmelCase):
					return int(x / 2**20)


class    _lowercase :
				'''simple docstring'''







				def __enter__(			self     ):

									'''simple docstring'''


									gc.collect()
									torch.cuda.empty_cache()
									torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
									UpperCamelCase_			= torch.cuda.memory_allocated()
									UpperCamelCase_			= psutil.Process()

									UpperCamelCase_			= self.cpu_mem_used()
									UpperCamelCase_			= True
									UpperCamelCase_			= threading.Thread(target=self.peak_monitor_func     )
									UpperCamelCase_			= True
									peak_monitor_thread.start()
									return self

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									return self.process.memory_info().rss

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= -1

									while True:
														UpperCamelCase_			= max(self.cpu_mem_used()  ,			self.cpu_peak     )

														# can't sleep or will not catch the peak right (this comment is here on purpose)
														# time.sleep(0.001) # 1msec

														if not self.peak_monitoring:
																			break

				def __exit__(			self  ,			*snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_			= False

									gc.collect()
									torch.cuda.empty_cache()
									UpperCamelCase_			= torch.cuda.memory_allocated()
									UpperCamelCase_			= torch.cuda.max_memory_allocated()
									UpperCamelCase_			= bamb(self.end - self.begin     )
									UpperCamelCase_			= bamb(self.peak - self.begin     )

									UpperCamelCase_			= self.cpu_mem_used()
									UpperCamelCase_			= bamb(self.cpu_end - self.cpu_begin     )
									UpperCamelCase_			= bamb(self.cpu_peak - self.cpu_begin     )
									# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
	from accelerate.test_utils.training import mocked_dataloaders

	UpperCAmelCase :				Union[str, Any]							=mocked_dataloaders  # noqa: F811



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase):
					# For testing only
					if os.environ.get("TESTING_MOCKED_DATALOADERS" ,						_lowerCAmelCase) == "1":
										UpperCamelCase_			= 2

					# New Code #
					# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
					UpperCamelCase_			= FullyShardedDataParallelPlugin(
					    state_dict_config=FullStateDictConfig(offload_to_cpu=_lowerCAmelCase ,						ranka_only=_lowerCAmelCase) ,						optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_lowerCAmelCase ,						ranka_only=_lowerCAmelCase) ,						)

					# Initialize accelerator
					if args.with_tracking:
										UpperCamelCase_			= Accelerator(
										    cpu=args.cpu ,						mixed_precision=args.mixed_precision ,						log_with="wandb" ,						project_dir=args.logging_dir ,						fsdp_plugin=_lowerCAmelCase ,						)
					else:
										UpperCamelCase_			= Accelerator(fsdp_plugin=_lowerCAmelCase)
					accelerator.print(accelerator.distributed_type)

					if hasattr(args.checkpointing_steps ,						"isdigit"):
										if args.checkpointing_steps == "epoch":
															UpperCamelCase_			= args.checkpointing_steps
										elif args.checkpointing_steps.isdigit():
															UpperCamelCase_			= int(args.checkpointing_steps)
										else:
															raise ValueError(
															    f"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.""")
					else:
										UpperCamelCase_			= None
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					UpperCamelCase_			= config["lr"]
					UpperCamelCase_			= int(config["num_epochs"])
					UpperCamelCase_			= int(config["seed"])
					UpperCamelCase_			= int(config["batch_size"])

					# We need to initialize the trackers we use, and also store our configuration
					if args.with_tracking:
										UpperCamelCase_			= vars(_lowerCAmelCase)
										accelerator.init_trackers("fsdp_glue_no_trainer" ,						_lowerCAmelCase)

					UpperCamelCase_			= AutoTokenizer.from_pretrained(args.model_name_or_path)
					UpperCamelCase_			= load_dataset("glue" ,						"mrpc")
					UpperCamelCase_			= evaluate.load("glue" ,						"mrpc")

					def tokenize_function(_lowerCAmelCase):
										# max_length=None => use the model max length (it's actually the default)
										UpperCamelCase_			= tokenizer(examples["sentence1"] ,						examples["sentence2"] ,						truncation=_lowerCAmelCase ,						max_length=_lowerCAmelCase)
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										UpperCamelCase_			= datasets.map(
										    _lowerCAmelCase ,						batched=_lowerCAmelCase ,						remove_columns=["idx", "sentence1", "sentence2"] ,						)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					UpperCamelCase_			= tokenized_datasets.rename_column("label" ,						"labels")

					# If the batch size is too big we use gradient accumulation
					UpperCamelCase_			= 1
					if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
										UpperCamelCase_			= batch_size // MAX_GPU_BATCH_SIZE
										UpperCamelCase_			= MAX_GPU_BATCH_SIZE

					def collate_fn(_lowerCAmelCase):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										UpperCamelCase_			= 1_28 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															UpperCamelCase_			= 16
										elif accelerator.mixed_precision != "no":
															UpperCamelCase_			= 8
										else:
															UpperCamelCase_			= None

										return tokenizer.pad(
										    _lowerCAmelCase ,						padding="longest" ,						max_length=_lowerCAmelCase ,						pad_to_multiple_of=_lowerCAmelCase ,						return_tensors="pt" ,						)

					# Instantiate dataloaders.
					UpperCamelCase_			= DataLoader(
					    tokenized_datasets["train"] ,						shuffle=_lowerCAmelCase ,						collate_fn=_lowerCAmelCase ,						batch_size=_lowerCAmelCase)
					UpperCamelCase_			= DataLoader(
					    tokenized_datasets["validation"] ,						shuffle=_lowerCAmelCase ,						collate_fn=_lowerCAmelCase ,						batch_size=_lowerCAmelCase)

					set_seed(_lowerCAmelCase)

					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					UpperCamelCase_			= AutoModelForSequenceClassification.from_pretrained(
					    args.model_name_or_path ,						return_dict=_lowerCAmelCase ,						low_cpu_mem_usage=_lowerCAmelCase)

					# New Code #
					# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
					UpperCamelCase_			= accelerator.prepare(_lowerCAmelCase)
					accelerator.print(_lowerCAmelCase)

					# Instantiate optimizer
					# New Code #
					# For FSDP feature, at present it doesn't support multiple parameter groups,
					# so we need to create a single parameter group for the whole model
					UpperCamelCase_			= torch.optim.AdamW(params=model.parameters() ,						lr=_lowerCAmelCase ,						weight_decay=2e-4)

					# Instantiate scheduler
					UpperCamelCase_			= get_linear_schedule_with_warmup(
					    optimizer=_lowerCAmelCase ,						num_warmup_steps=10 ,						num_training_steps=(len(_lowerCAmelCase) * num_epochs) // gradient_accumulation_steps ,						)

					# New Code #
					# For FSDP feature, prepare everything except the model as we have already prepared the model
					# before creating the optimizer
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_			= accelerator.prepare(
					    _lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase)

					UpperCamelCase_			= 0

					# Potentially load in the weights and states from a previous save
					if args.resume_from_checkpoint:
										if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
															accelerator.print(f"""Resumed from checkpoint: {args.resume_from_checkpoint}""")
															accelerator.load_state(args.resume_from_checkpoint)
															UpperCamelCase_			= os.path.basename(args.resume_from_checkpoint)
										else:
															# Get the most recent checkpoint
															UpperCamelCase_			= [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]
															dirs.sort(key=os.path.getctime)
															UpperCamelCase_			= dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
										# Extract `epoch_{i}` or `step_{i}`
										UpperCamelCase_			= os.path.splitext(_lowerCAmelCase)[0]

										if "epoch" in training_difference:
															num_epochs -= int(training_difference.replace("epoch_" ,						""))
															UpperCamelCase_			= None
										else:
															UpperCamelCase_			= int(training_difference.replace("step_" ,						""))
															num_epochs -= resume_step // len(_lowerCAmelCase)
															# If resuming by step, we also need to know exactly how far into the DataLoader we went
															UpperCamelCase_			= (num_epochs * len(_lowerCAmelCase)) - resume_step

    # Now we train the model
					for epoch in range(_lowerCAmelCase):
										# New Code #
										# context manager to track the peak memory usage during the training epoch
										with TorchTracemalloc() as tracemalloc:
															model.train()
															if args.with_tracking:
																				UpperCamelCase_			= 0
															for step, batch in enumerate(_lowerCAmelCase):
																				# We need to skip steps until we reach the resumed step
																				if args.resume_from_checkpoint and epoch == 0:
																									if resume_step is not None and step < resume_step:
																														pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device)
																				UpperCamelCase_			= model(**_lowerCAmelCase)
																				UpperCamelCase_			= outputs.loss
																				UpperCamelCase_			= loss / gradient_accumulation_steps
																				# We keep track of the loss at each epoch
																				if args.with_tracking:
																									total_loss += loss.detach().float()
																				accelerator.backward(_lowerCAmelCase)
																				if step % gradient_accumulation_steps == 0:
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# accelerator.print(lr_scheduler.get_lr())

																				overall_step += 1

																				if isinstance(_lowerCAmelCase ,						_lowerCAmelCase):
																									UpperCamelCase_			= f"""step_{overall_step}"""
																									if overall_step % checkpointing_steps == 0:
																														if args.output_dir is not None:
																																			UpperCamelCase_			= os.path.join(args.output_dir ,						_lowerCAmelCase)
																														accelerator.save_state(_lowerCAmelCase)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin)))
										accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used))
										accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked))
										accelerator.print(
										    "Total Peak Memory consumed during the train (max): {}".format(
										        tracemalloc.peaked + bamb(tracemalloc.begin)))
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        "train_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin),
															    } ,						step=_lowerCAmelCase ,						)

										# New Code #
										# context manager to track the peak memory usage during the evaluation
										with TorchTracemalloc() as tracemalloc:
															model.eval()
															for step, batch in enumerate(_lowerCAmelCase):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device)
																				with torch.no_grad():
																									UpperCamelCase_			= model(**_lowerCAmelCase)
																				UpperCamelCase_			= outputs.logits.argmax(dim=-1)
																				UpperCamelCase_	,     UpperCamelCase_			= accelerator.gather_for_metrics((predictions, batch["labels"]))
																				metric.add_batch(
																				    predictions=_lowerCAmelCase ,						references=_lowerCAmelCase ,						)

															UpperCamelCase_			= metric.compute()
															# Use accelerator.print to print only on the main process.
															accelerator.print(f"""epoch {epoch}:""" ,						_lowerCAmelCase)
															if args.with_tracking:
																				accelerator.log(
																				    {
																				        "accuracy": eval_metric["accuracy"],
																				        "f1": eval_metric["f1"],
																				        "train_loss": total_loss.item() / len(_lowerCAmelCase),
																				    } ,						step=_lowerCAmelCase ,						)

															if checkpointing_steps == "epoch":
																				UpperCamelCase_			= f"""epoch_{epoch}"""
																				if args.output_dir is not None:
																									UpperCamelCase_			= os.path.join(args.output_dir ,						_lowerCAmelCase)
																				accelerator.save_state(_lowerCAmelCase)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the eval : {}".format(bamb(tracemalloc.begin)))
										accelerator.print("Memory consumed at the end of the eval (end-begin): {}".format(tracemalloc.used))
										accelerator.print("Peak Memory consumed during the eval (max-begin): {}".format(tracemalloc.peaked))
										accelerator.print(
										    "Total Peak Memory consumed during the eval (max): {}".format(tracemalloc.peaked + bamb(tracemalloc.begin)))
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        "eval_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin),
															    } ,						step=_lowerCAmelCase ,						)

					if args.with_tracking:
										accelerator.end_training()



def  _lowerCAmelCase     ():
					UpperCamelCase_			= argparse.ArgumentParser(description="Simple example of training script.")
					parser.add_argument(
					    "--mixed_precision" ,						type=_lowerCAmelCase ,						default=_lowerCAmelCase ,						choices=["no", "fp16", "bf16", "fp8"] ,						help="Whether to use mixed precision. Choose"
					    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
					    "and an Nvidia Ampere GPU." ,						)
					parser.add_argument("--cpu" ,						action="store_true" ,						help="If passed, will train on the CPU.")
					parser.add_argument(
					    "--checkpointing_steps" ,						type=_lowerCAmelCase ,						default=_lowerCAmelCase ,						help="Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch." ,						)
					parser.add_argument(
					    "--resume_from_checkpoint" ,						type=_lowerCAmelCase ,						default=_lowerCAmelCase ,						help="If the training should continue from a checkpoint folder." ,						)
					parser.add_argument(
					    "--with_tracking" ,						action="store_true" ,						help="Whether to load in all available experiment trackers from the environment and use them for logging." ,						)
					parser.add_argument(
					    "--output_dir" ,						type=_lowerCAmelCase ,						default="." ,						help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory." ,						)
					parser.add_argument(
					    "--logging_dir" ,						type=_lowerCAmelCase ,						default="logs" ,						help="Location on where to store experiment tracking logs`" ,						)
					parser.add_argument(
					    "--model_name_or_path" ,						type=_lowerCAmelCase ,						help="Path to pretrained model or model identifier from huggingface.co/models." ,						required=_lowerCAmelCase ,						)
					UpperCamelCase_			= parser.parse_args()
					UpperCamelCase_			= {"lr": 2e-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
					training_function(_lowerCAmelCase ,						_lowerCAmelCase)


if __name__ == "__main__":
	main()

"""simple docstring"""


import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MarkupLMForQuestionAnswering,
     MarkupLMForSequenceClassification,
     MarkupLMForTokenClassification,
     MarkupLMModel,
 )

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer






class 					a					:



  def __init__(		self   :    str						,  lowerCamelCase_   :    List[str]						,  lowerCamelCase_   :    Tuple=13						,  lowerCamelCase_   :    List[Any]=7						,  lowerCamelCase_   :    List[str]=True						,  lowerCamelCase_   :    str=True						,  lowerCamelCase_   :    Tuple=True						,  lowerCamelCase_   :    Optional[Any]=True						,  lowerCamelCase_   :    str=99						,  lowerCamelCase_   :    str=32						,  lowerCamelCase_   :    List[Any]=5						,  lowerCamelCase_   :    Dict=4						,  lowerCamelCase_   :    str=37						,  lowerCamelCase_   :    List[Any]="gelu"						,  lowerCamelCase_   :    List[str]=0.1						,  lowerCamelCase_   :    Optional[int]=0.1						,  lowerCamelCase_   :    List[str]=5_12						,  lowerCamelCase_   :    Union[str, Any]=16						,  lowerCamelCase_   :    Any=2						,  lowerCamelCase_   :    List[str]=0.02						,  lowerCamelCase_   :    List[Any]=3						,  lowerCamelCase_   :    Any=None						,  lowerCamelCase_   :    List[Any]=20						,  lowerCamelCase_   :    Tuple=30						,  lowerCamelCase_   :    Optional[Any]=2						,  lowerCamelCase_   :    List[Any]=2						,  lowerCamelCase_   :    Union[str, Any]=10						,  )      -> Any:
      __a			      =						parent
      __a			      =						batch_size
      __a			      =						seq_length
      __a			      =						is_training
      __a			      =						use_input_mask
      __a			      =						use_token_type_ids
      __a			      =						use_labels
      __a			      =						vocab_size
      __a			      =						hidden_size
      __a			      =						num_hidden_layers
      __a			      =						num_attention_heads
      __a			      =						intermediate_size
      __a			      =						hidden_act
      __a			      =						hidden_dropout_prob
      __a			      =						attention_probs_dropout_prob
      __a			      =						max_position_embeddings
      __a			      =						type_vocab_size
      __a			      =						type_sequence_label_size
      __a			      =						initializer_range
      __a			      =						num_labels
      __a			      =						scope
      __a			      =						max_xpath_tag_unit_embeddings
      __a			      =						max_xpath_subs_unit_embeddings
      __a			      =						tag_pad_id
      __a			      =						subs_pad_id
      __a			      =						max_depth



  def        lowerCAmelCase_				(		self   :    Dict     )      -> Dict:
      __a			      =						ids_tensor([self.batch_size, self.seq_length]						,  self.vocab_size     )

      __a			      =						ids_tensor(
          [self.batch_size, self.seq_length, self.max_depth]						,  self.max_xpath_tag_unit_embeddings     )

      __a			      =						ids_tensor(
          [self.batch_size, self.seq_length, self.max_depth]						,  self.max_xpath_subs_unit_embeddings     )

      __a			      =						None
      if self.use_input_mask:
          __a			      =						ids_tensor([self.batch_size, self.seq_length]						,  vocab_size=2     )

      __a			      =						None
      if self.use_token_type_ids:
          __a			      =						ids_tensor([self.batch_size, self.seq_length]						,  self.type_vocab_size     )

      __a			      =						None
      __a			      =						None
      if self.use_labels:
          __a			      =						ids_tensor([self.batch_size]						,  self.type_sequence_label_size     )
          __a			      =						ids_tensor([self.batch_size, self.seq_length]						,  self.num_labels     )

      __a			      =						self.get_config()

      return (
          config,
          input_ids,
          xpath_tags_seq,
          xpath_subs_seq,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
      )



  def        lowerCAmelCase_				(		self   :    List[Any]     )      -> Optional[Any]:
      return MarkupLMConfig(
          vocab_size=self.vocab_size						,  hidden_size=self.hidden_size						,  num_hidden_layers=self.num_hidden_layers						,  num_attention_heads=self.num_attention_heads						,  intermediate_size=self.intermediate_size						,  hidden_act=self.hidden_act						,  hidden_dropout_prob=self.hidden_dropout_prob						,  attention_probs_dropout_prob=self.attention_probs_dropout_prob						,  max_position_embeddings=self.max_position_embeddings						,  type_vocab_size=self.type_vocab_size						,  initializer_range=self.initializer_range						,  max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings						,  max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings						,  tag_pad_id=self.tag_pad_id						,  subs_pad_id=self.subs_pad_id						,  max_depth=self.max_depth						,  )



  def        lowerCAmelCase_				(		self   :    Tuple						,  lowerCamelCase_   :    int						,  lowerCamelCase_   :    int						,  lowerCamelCase_   :    List[str]						,  lowerCamelCase_   :    int						,  lowerCamelCase_   :    Any						,  lowerCamelCase_   :    Optional[Any]						,  lowerCamelCase_   :    Any						,  lowerCamelCase_   :    List[Any]						,  )      -> int:
      __a			      =						MarkupLMModel(config=lowerCamelCase_     )
      model.to(lowerCamelCase_     )
      model.eval()
      print("""Configs:"""						,  model.config.tag_pad_id						,  model.config.subs_pad_id     )
      __a			      =						model(lowerCamelCase_						,  attention_mask=lowerCamelCase_						,  token_type_ids=lowerCamelCase_     )
      __a			      =						model(lowerCamelCase_						,  token_type_ids=lowerCamelCase_     )
      __a			      =						model(lowerCamelCase_     )
      self.parent.assertEqual(result.last_hidden_state.shape						,  (self.batch_size, self.seq_length, self.hidden_size)     )
      self.parent.assertEqual(result.pooler_output.shape						,  (self.batch_size, self.hidden_size)     )



  def        lowerCAmelCase_				(		self   :    Optional[int]						,  lowerCamelCase_   :    List[Any]						,  lowerCamelCase_   :    Optional[int]						,  lowerCamelCase_   :    str						,  lowerCamelCase_   :    Optional[int]						,  lowerCamelCase_   :    List[Any]						,  lowerCamelCase_   :    Tuple						,  lowerCamelCase_   :    List[str]						,  lowerCamelCase_   :    int						,  )      -> Optional[Any]:
      __a			      =						self.num_labels
      __a			      =						MarkupLMForSequenceClassification(lowerCamelCase_     )
      model.to(lowerCamelCase_     )
      model.eval()
      __a			      =						model(
          lowerCamelCase_						,  xpath_tags_seq=lowerCamelCase_						,  xpath_subs_seq=lowerCamelCase_						,  attention_mask=lowerCamelCase_						,  token_type_ids=lowerCamelCase_						,  labels=lowerCamelCase_						,  )
      self.parent.assertEqual(result.logits.shape						,  (self.batch_size, self.num_labels)     )



  def        lowerCAmelCase_				(		self   :    int						,  lowerCamelCase_   :    Any						,  lowerCamelCase_   :    Any						,  lowerCamelCase_   :    List[Any]						,  lowerCamelCase_   :    Dict						,  lowerCamelCase_   :    Union[str, Any]						,  lowerCamelCase_   :    Union[str, Any]						,  lowerCamelCase_   :    str						,  lowerCamelCase_   :    Union[str, Any]						,  )      -> int:
      __a			      =						self.num_labels
      __a			      =						MarkupLMForTokenClassification(config=lowerCamelCase_     )
      model.to(lowerCamelCase_     )
      model.eval()
      __a			      =						model(
          lowerCamelCase_						,  xpath_tags_seq=lowerCamelCase_						,  xpath_subs_seq=lowerCamelCase_						,  attention_mask=lowerCamelCase_						,  token_type_ids=lowerCamelCase_						,  labels=lowerCamelCase_						,  )
      self.parent.assertEqual(result.logits.shape						,  (self.batch_size, self.seq_length, self.num_labels)     )



  def        lowerCAmelCase_				(		self   :    List[Any]						,  lowerCamelCase_   :    Tuple						,  lowerCamelCase_   :    Dict						,  lowerCamelCase_   :    Optional[int]						,  lowerCamelCase_   :    Optional[Any]						,  lowerCamelCase_   :    Optional[Any]						,  lowerCamelCase_   :    Union[str, Any]						,  lowerCamelCase_   :    Dict						,  lowerCamelCase_   :    List[Any]						,  )      -> Tuple:
      __a			      =						MarkupLMForQuestionAnswering(config=lowerCamelCase_     )
      model.to(lowerCamelCase_     )
      model.eval()
      __a			      =						model(
          lowerCamelCase_						,  xpath_tags_seq=lowerCamelCase_						,  xpath_subs_seq=lowerCamelCase_						,  attention_mask=lowerCamelCase_						,  token_type_ids=lowerCamelCase_						,  start_positions=lowerCamelCase_						,  end_positions=lowerCamelCase_						,  )
      self.parent.assertEqual(result.start_logits.shape						,  (self.batch_size, self.seq_length)     )
      self.parent.assertEqual(result.end_logits.shape						,  (self.batch_size, self.seq_length)     )



  def        lowerCAmelCase_				(		self   :    str     )      -> str:
      __a			      =						self.prepare_config_and_inputs()
      (
          (
          __a
      )      ,  (
          __a
      )      ,  (
          __a
      )      ,  (
          __a
      )      ,  (
          __a
      )      ,  (
          __a
      )      ,  (
          __a
      )      ,  (
          __a
      )      ,  
      )			      =						config_and_inputs
      __a			      =						{
          """input_ids""": input_ids,
          """xpath_tags_seq""": xpath_tags_seq,
          """xpath_subs_seq""": xpath_subs_seq,
          """token_type_ids""": token_type_ids,
          """attention_mask""": input_mask,
      }
      return config, inputs_dict






@require_torch
class 					a					(				A_   ,		A_   ,		unittest.TestCase							):
  A_ :				int 			=							(
      (
          MarkupLMModel,
          MarkupLMForSequenceClassification,
          MarkupLMForTokenClassification,
          MarkupLMForQuestionAnswering,
      )
      if is_torch_available()
      else None
  )
  A_ :				Any 			=							(
      {
          '''feature-extraction''': MarkupLMModel,
          '''question-answering''': MarkupLMForQuestionAnswering,
          '''text-classification''': MarkupLMForSequenceClassification,
          '''token-classification''': MarkupLMForTokenClassification,
          '''zero-shot''': MarkupLMForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )



  def        lowerCAmelCase_				(		self   :    Union[str, Any]						,  lowerCamelCase_   :    Optional[Any]						,  lowerCamelCase_   :    Union[str, Any]						,  lowerCamelCase_   :    Optional[int]						,  lowerCamelCase_   :    List[str]						,  lowerCamelCase_   :    Union[str, Any]     )      -> List[Any]:
      # ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
      # (batch of pretokenized examples).
      return True



  def        lowerCAmelCase_				(		self   :    Tuple     )      -> str:
      __a			      =						MarkupLMModelTester(self     )
      __a			      =						ConfigTester(self						,  config_class=lowerCamelCase_						,  hidden_size=37     )



  def        lowerCAmelCase_				(		self   :    Union[str, Any]     )      -> List[str]:
      self.config_tester.run_common_tests()



  def        lowerCAmelCase_				(		self   :    str     )      -> Dict:
      __a			      =						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowerCamelCase_     )



  def        lowerCAmelCase_				(		self   :    Dict     )      -> Optional[int]:
      __a			      =						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_     )



  def        lowerCAmelCase_				(		self   :    Tuple     )      -> Tuple:
      __a			      =						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*lowerCamelCase_     )



  def        lowerCAmelCase_				(		self   :    Union[str, Any]     )      -> int:
      __a			      =						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_     )

def 						UpperCamelCase				(	):
    __a			      =						"""
    <!DOCTYPE html>
    <html>
    <head>
    <title>Page Title</title>
    </head>
    <body>

    <h1>This is a Heading</h1>
    <p>This is a paragraph.</p>

    </body>
    </html>
    """

    return html_string






@require_torch
class 					a					(				unittest.TestCase							):



  @cached_property
  def        lowerCAmelCase_				(		self   :    int     )      -> Any:
      # TODO use from_pretrained here
      __a			      =						MarkupLMFeatureExtractor()
      __a			      =						MarkupLMTokenizer.from_pretrained("""microsoft/markuplm-base"""     )

      return MarkupLMProcessor(lowerCamelCase_						,  lowerCamelCase_     )



  @slow
  def        lowerCAmelCase_				(		self   :    Optional[int]     )      -> Dict:
      __a			      =						MarkupLMModel.from_pretrained("""microsoft/markuplm-base"""     ).to(lowerCamelCase_     )

      __a			      =						self.default_processor

      __a			      =						processor(prepare_html_string()						,  return_tensors="""pt"""     )
      __a			      =						inputs.to(lowerCamelCase_     )

      # forward pass
      with torch.no_grad():
          __a			      =						model(**lowerCamelCase_     )

      # verify the last hidden states
      __a			      =						torch.Size([1, 14, 7_68]     )
      self.assertEqual(outputs.last_hidden_state.shape						,  lowerCamelCase_     )

      __a			      =						torch.tensor(
          [[0.06_75, -0.00_52, 0.50_01], [-0.22_81, 0.08_02, 0.21_92], [-0.05_83, -0.33_11, 0.11_85]]     ).to(lowerCamelCase_     )

      self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]						,  lowerCamelCase_						,  atol=1E-4     )     )
