code: 
  
  
  import argparse
  
  # utilities to import the model weights and config file
  import os
  import pickle as pkl
  
  # PyTorch + new model classes
  import torch
  from torch import nn
  
  from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM
  
  
  # import the EncoderLayer class used to pretrain
  # !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
  try:
   from fairseq.modules.mega_layer import MegaEncoderLayer
  except ImportError:
   raise ImportError('You need to install the version of fairseq from the Mega repo!')
  
  
  class 		_SCREAMING_SNAKE_CASE    (	nn.Module    ):
  
  
       def __init__(     self       ,				A_       ,				A_       ,				A_						):
         super().__init__()
         _UpperCAmelCase		:       Dict       =    mega_args
         _UpperCAmelCase		:       Dict       =    nn.Embedding(A_       ,				self.mega_args.encoder_embed_dim						)
         _UpperCAmelCase		:       Tuple       =    nn.ModuleList([MegaEncoderLayer(self.mega_args						) for _ in range(A_						)]						)
         _UpperCAmelCase		:       Optional[int]       =    depth
  
  
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_=True       ,				A_=0						):
  
         # Mega expects embeddings to be (time, batch, embedding size), but
         # Hugging Face returns tokens as (batch, time)
         if batch_first:
           _UpperCAmelCase		:       Optional[Any]       =    input_ids.T
  
         # to make things more confusing, Mega expects the attention mask to
         # be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
         # which is the opposite of what HF returns
         if ignore_mask_value == 0:
           _UpperCAmelCase		:       int       =    1 - attention_mask
  
         # get token embeddings from IDs
         _UpperCAmelCase		:       Union[str, Any]       =    self.embedding_layer(A_						)
  
         # pass through the Mega layers
         # input is (time, batch, encoder dim) and output is the same
         for encoder in self.encoders:
           _UpperCAmelCase		:       List[str]       =    encoder(A_       ,				A_						)
  
         # return according to the shape specified
         if batch_first:
           # (T, B, H) --> (B, T, H)
           return torch.transpose(A_       ,				0       ,				1						)
         else:
           return embeds
  
  
  class 		_SCREAMING_SNAKE_CASE    (	nn.Module    ):
  
  
       def __init__(     self       ,				A_       ,				A_       ,				A_						):
         super().__init__()
         _UpperCAmelCase		:       Any       =    MegaLM(A_       ,				A_       ,				A_						)
         _UpperCAmelCase		:       Any       =    nn.Linear(mega_args.encoder_embed_dim       ,				A_						)
         _UpperCAmelCase		:       Dict       =    nn.Dropout(p=0.1						)
  
  
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_=True       ,				A_=0						):
         _UpperCAmelCase		:       int       =    self.mega(A_       ,				A_       ,				A_       ,				A_						)
         return self.mlm_head(self.dropout(A_						)						)
  
  
  
  
  def       a__						(		snake_case__				:       List[Any]	,      snake_case__				:       str	,      snake_case__				:       List[str]    ):
    with open(os.path.join(snake_case__	,      """model_args.pkl"""    )	,      """rb"""    ) as f:
      _UpperCAmelCase		:       Union[str, Any]       =    pkl.load(snake_case__    )
  
    # load the original encoder
    _UpperCAmelCase		:       int       =    OriginalMegaForMaskedLM(**snake_case__    ).eval()
  
    # load its weights
    print(
        """Original Mega encoder:"""	,      original_mlm.mega.load_state_dict(
            torch.load(os.path.join(snake_case__	,      """encoder_weights.pt"""    )	,      map_location="""cpu"""    )    )	,      )
    print(
        """Original Mega MLM layer:"""	,      original_mlm.mlm_head.load_state_dict(
            torch.load(os.path.join(snake_case__	,      """mlm_head_weights.pt"""    )	,      map_location="""cpu"""    )    )	,      )
  
    # create a new config from the old one
    _UpperCAmelCase		:       List[str]       =    MegaConfig(
        num_hidden_layers=mega_original_args["""depth"""]	,      vocab_size=mega_original_args["""vocab_size"""]	,      hidden_size=mega_original_args["""mega_args"""].encoder_embed_dim	,      shared_representation_size=mega_original_args["""mega_args"""].encoder_z_dim	,      intermediate_size=mega_original_args["""mega_args"""].encoder_hidden_dim	,      ema_projection_size=mega_original_args["""mega_args"""].encoder_n_dim	,      dropout_prob=mega_original_args["""mega_args"""].dropout	,      attention_probs_dropout_prob=mega_original_args["""mega_args"""].attention_dropout	,      hidden_dropout_prob=mega_original_args["""mega_args"""].hidden_dropout	,      activation=mega_original_args["""mega_args"""].activation_fn	,      attention_activation=mega_original_args["""mega_args"""].attention_activation_fn	,      bidirectional=mega_original_args["""mega_args"""].bidirectional	,      use_chunking=mega_original_args["""mega_args"""].encoder_chunk_size > 0	,      chunk_size=mega_original_args["""mega_args"""].encoder_chunk_size	,      truncation=mega_original_args["""mega_args"""].truncation_length	,      normalization_type=mega_original_args["""mega_args"""].normalization_type	,      normalize_before_mega=snake_case__	,      norm_affine=snake_case__	,      use_feature_dropout=mega_original_args["""mega_args"""].feature_dropout	,      relative_positional_bias=mega_original_args["""mega_args"""].rel_pos_bias	,      max_positions=mega_original_args["""mega_args"""].max_source_positions	,      nffn_hidden_size=mega_original_args["""mega_args"""].encoder_ffn_embed_dim	,      normalize_before_ffn=mega_original_args["""mega_args"""].normalize_before	,      nffn_activation_dropout_prob=0.0	,      add_token_type_embeddings=snake_case__	,      add_lm_hidden_dense_layer=snake_case__	,      )
  
    _UpperCAmelCase		:       Any       =    MegaForMaskedLM(snake_case__    ).eval()
  
    # the originl checkpoint just uses nn.Embedding for the word embeddings
    # we use a wrapper module for embeddings to add support for positional embeddings
    _UpperCAmelCase		:       Dict       =    original_mlm.mega.embedding_layer.weight
  
    # modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
    # ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
    # also renaming previously confusing parameter names
    _UpperCAmelCase		:       List[str]       =    original_mlm.mega.encoders.state_dict()
    _UpperCAmelCase		:       List[str]       =    {}
    for module_name in original_state_dict.keys():
      _UpperCAmelCase		:       Any       =    None
      # have to handle gamma, beta, and alpha differently due to their use
      # in multiple modules within the original repository;
      # beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
      # the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
      if "beta" in module_name:
        # EMA sub-layers were always called "move" in the original repo
        if "move.beta" in module_name:
          _UpperCAmelCase		:       List[Any]       =    module_name.replace("""move.beta"""	,      """ema_gate.ema_expansion_matrix"""    )
        elif "mega_layer.beta" in module_name:
          _UpperCAmelCase		:       Dict       =    module_name.replace("""beta"""	,      """qk_bias"""    )
        else:
          _UpperCAmelCase		:       Tuple       =    module_name.replace("""beta"""	,      """b_param"""    )
          # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
      elif "gamma" in module_name:
        if "move.gamma" in module_name:
          _UpperCAmelCase		:       Any       =    module_name.replace("""move.gamma"""	,      """ema_gate.kernel_projection_matrix"""    )
        elif "mega_layer.gamma" in module_name:
          _UpperCAmelCase		:       List[str]       =    module_name.replace("""gamma"""	,      """qk_weight"""    )
        else:
          _UpperCAmelCase		:       List[str]       =    module_name.replace("""gamma"""	,      """g_param"""    )
          # alpha is used in EMA and positional bias; renaming to improve readability
      elif "move.alpha" in module_name:
        _UpperCAmelCase		:       Any       =    module_name.replace("""move.alpha"""	,      """ema_gate.decay_factor"""    )
      # delta is only used in EMA; renaming to improve readability
      elif "move.delta" in module_name:
        _UpperCAmelCase		:       Tuple       =    module_name.replace("""move.delta"""	,      """ema_gate.damping_factor"""    )
      # omega is only used in EMA; renaming to improve readability
      elif "omega" in module_name:
        _UpperCAmelCase		:       Union[str, Any]       =    module_name.replace("""move.omega"""	,      """ema_gate.residual_weight"""    )
  
      if new_module_name:
        _UpperCAmelCase		:       Dict       =    new_module_name
  
    if len(snake_case__    ) != 0:
      print(f'''Renaming these keys: {updated_keys.keys()}'''    )
    else:
      print("""No need to rename state dict entries"""    )
    for old, new in updated_keys.items():
      _UpperCAmelCase		:       List[Any]       =    original_state_dict.pop(snake_case__    )
  
    # now attempt to load the state dictionary with updated names
    # note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
    print("""HF Mega encoder:"""	,      hf_mlm.mega.layers.load_state_dict(snake_case__    )    )
  
    # load the MLM head weights directly
    print(
        """HF Mega MLM layer:"""	,      hf_mlm.mlm_head.load_state_dict(
            torch.load(os.path.join(snake_case__	,      """mlm_head_weights.pt"""    )	,      map_location="""cpu"""    )    )	,      )
  
    # test on a randomly generated input sequence
    _UpperCAmelCase		:       Dict       =    torch.randint(0	,      hf_config.vocab_size	,      size=(4, 256)    )
    _UpperCAmelCase		:       Any       =    torch.ones_like(snake_case__    )
    # mask a few tokens to make sure masking is applied appropriately :)
    _UpperCAmelCase		:       Dict       =    0
  
    # run forward passes
    _UpperCAmelCase		:       List[Any]       =    original_mlm(snake_case__	,      snake_case__	,      batch_first=snake_case__	,      ignore_mask_value=0    )
    _UpperCAmelCase		:       Union[str, Any]       =    hf_mlm(snake_case__	,      snake_case__    )[0]
  
    # print shapes and diff
    print(f'''original output {original_output.shape}'''    )
    print(f'''hf output {hf_output.shape}'''    )
    print(f'''max diff: {(original_output - hf_output).max()}'''    )  # 0.0
    _UpperCAmelCase		:       List[Any]       =    torch.allclose(snake_case__	,      snake_case__	,      atol=1e-3    )
  
    if success:
      print("""Yay!"""    )
      hf_mlm.save_pretrained(snake_case__    )
    else:
      raise RuntimeError(f'''Something\'s broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}'''    )
  
    if includes_tokenizer:
      print("""Transferring tokenizer"""    )
      _UpperCAmelCase		:       Optional[int]       =    AutoTokenizer.from_pretrained(snake_case__    )
      tokenizer.save_pretrained(snake_case__    )
  
  
  if __name__ == "__main__":
   SCREAMING_SNAKE_CASE__				:    Optional[int]         =    argparse.ArgumentParser()
  
   parser.add_argument(
       '--pretrained_checkpoint_path',
       default=None,
       type=str,
       required=True,
       help='Point to the directory containing your model weights using the official Mega repo',
   )
  
   parser.add_argument(
       '--output_path', default=None, type=str, required=True, help='Location to save the Hugging Face version'
   )
  
   parser.add_argument(
       '--includes_tokenizer',
       action='store_true',
       help='Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo',
   )
  
   SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    parser.parse_args()
  
   convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)
  
  
  
  
  
  
style_context: 
  
  
  import unittest
  
  from transformers import MarkupLMConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  from transformers.utils import cached_property
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
   import torch
  
   from transformers import (
       MarkupLMForQuestionAnswering,
       MarkupLMForSequenceClassification,
       MarkupLMForTokenClassification,
       MarkupLMModel,
   )
  
  # TODO check dependencies
  from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer
  
  
  class 		_SCREAMING_SNAKE_CASE    :
  
  
       def __init__(     self       ,				A_       ,				A_=13       ,				A_=7       ,				A_=True       ,				A_=True       ,				A_=True       ,				A_=True       ,				A_=99       ,				A_=32       ,				A_=5       ,				A_=4       ,				A_=37       ,				A_="gelu"       ,				A_=0.1       ,				A_=0.1       ,				A_=5_12       ,				A_=16       ,				A_=2       ,				A_=0.0_2       ,				A_=3       ,				A_=None       ,				A_=20       ,				A_=30       ,				A_=2       ,				A_=2       ,				A_=10       ,				):
         _UpperCAmelCase		:       str       =    parent
         _UpperCAmelCase		:       Dict       =    batch_size
         _UpperCAmelCase		:       Optional[int]       =    seq_length
         _UpperCAmelCase		:       Union[str, Any]       =    is_training
         _UpperCAmelCase		:       Tuple       =    use_input_mask
         _UpperCAmelCase		:       Dict       =    use_token_type_ids
         _UpperCAmelCase		:       Dict       =    use_labels
         _UpperCAmelCase		:       Optional[int]       =    vocab_size
         _UpperCAmelCase		:       List[str]       =    hidden_size
         _UpperCAmelCase		:       Tuple       =    num_hidden_layers
         _UpperCAmelCase		:       Tuple       =    num_attention_heads
         _UpperCAmelCase		:       Optional[int]       =    intermediate_size
         _UpperCAmelCase		:       str       =    hidden_act
         _UpperCAmelCase		:       Optional[Any]       =    hidden_dropout_prob
         _UpperCAmelCase		:       List[str]       =    attention_probs_dropout_prob
         _UpperCAmelCase		:       Union[str, Any]       =    max_position_embeddings
         _UpperCAmelCase		:       List[Any]       =    type_vocab_size
         _UpperCAmelCase		:       Union[str, Any]       =    type_sequence_label_size
         _UpperCAmelCase		:       List[Any]       =    initializer_range
         _UpperCAmelCase		:       Union[str, Any]       =    num_labels
         _UpperCAmelCase		:       Any       =    scope
         _UpperCAmelCase		:       List[str]       =    max_xpath_tag_unit_embeddings
         _UpperCAmelCase		:       Tuple       =    max_xpath_subs_unit_embeddings
         _UpperCAmelCase		:       Optional[Any]       =    tag_pad_id
         _UpperCAmelCase		:       Union[str, Any]       =    subs_pad_id
         _UpperCAmelCase		:       Optional[int]       =    max_depth
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Dict       =    ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size						)
  
         _UpperCAmelCase		:       List[str]       =    ids_tensor(
             [self.batch_size, self.seq_length, self.max_depth]       ,				self.max_xpath_tag_unit_embeddings						)
  
         _UpperCAmelCase		:       Tuple       =    ids_tensor(
             [self.batch_size, self.seq_length, self.max_depth]       ,				self.max_xpath_subs_unit_embeddings						)
  
         _UpperCAmelCase		:       int       =    None
         if self.use_input_mask:
           _UpperCAmelCase		:       Any       =    ids_tensor([self.batch_size, self.seq_length]       ,				vocab_size=2						)
  
         _UpperCAmelCase		:       Tuple       =    None
         if self.use_token_type_ids:
           _UpperCAmelCase		:       Dict       =    ids_tensor([self.batch_size, self.seq_length]       ,				self.type_vocab_size						)
  
         _UpperCAmelCase		:       str       =    None
         _UpperCAmelCase		:       Union[str, Any]       =    None
         if self.use_labels:
           _UpperCAmelCase		:       int       =    ids_tensor([self.batch_size]       ,				self.type_sequence_label_size						)
           _UpperCAmelCase		:       int       =    ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels						)
  
         _UpperCAmelCase		:       List[Any]       =    self.get_config()
  
         return (
             config,
             input_ids,
             xpath_tags_seq,
             xpath_subs_seq,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
         )
  
  
       def 					__snake_case(     self						):
         return MarkupLMConfig(
             vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				type_vocab_size=self.type_vocab_size       ,				initializer_range=self.initializer_range       ,				max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings       ,				max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings       ,				tag_pad_id=self.tag_pad_id       ,				subs_pad_id=self.subs_pad_id       ,				max_depth=self.max_depth       ,				)
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				):
         _UpperCAmelCase		:       Tuple       =    MarkupLMModel(config=A_						)
         model.to(A_						)
         model.eval()
         print("""Configs:"""       ,				model.config.tag_pad_id       ,				model.config.subs_pad_id						)
         _UpperCAmelCase		:       List[Any]       =    model(A_       ,				attention_mask=A_       ,				token_type_ids=A_						)
         _UpperCAmelCase		:       Tuple       =    model(A_       ,				token_type_ids=A_						)
         _UpperCAmelCase		:       Union[str, Any]       =    model(A_						)
         self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)						)
         self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)						)
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				):
         _UpperCAmelCase		:       Optional[int]       =    self.num_labels
         _UpperCAmelCase		:       List[Any]       =    MarkupLMForSequenceClassification(A_						)
         model.to(A_						)
         model.eval()
         _UpperCAmelCase		:       Any       =    model(
             A_       ,				xpath_tags_seq=A_       ,				xpath_subs_seq=A_       ,				attention_mask=A_       ,				token_type_ids=A_       ,				labels=A_       ,				)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)						)
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				):
         _UpperCAmelCase		:       List[Any]       =    self.num_labels
         _UpperCAmelCase		:       Any       =    MarkupLMForTokenClassification(config=A_						)
         model.to(A_						)
         model.eval()
         _UpperCAmelCase		:       Tuple       =    model(
             A_       ,				xpath_tags_seq=A_       ,				xpath_subs_seq=A_       ,				attention_mask=A_       ,				token_type_ids=A_       ,				labels=A_       ,				)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)						)
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_       ,				):
         _UpperCAmelCase		:       List[Any]       =    MarkupLMForQuestionAnswering(config=A_						)
         model.to(A_						)
         model.eval()
         _UpperCAmelCase		:       List[Any]       =    model(
             A_       ,				xpath_tags_seq=A_       ,				xpath_subs_seq=A_       ,				attention_mask=A_       ,				token_type_ids=A_       ,				start_positions=A_       ,				end_positions=A_       ,				)
         self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)						)
         self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)						)
  
  
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Union[str, Any]       =    self.prepare_config_and_inputs()
         (
             (
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),(
             _UpperCAmelCase
         ),
         )		:       List[str]       =    config_and_inputs
         _UpperCAmelCase		:       Optional[Any]       =    {
             """input_ids""": input_ids,
             """xpath_tags_seq""": xpath_tags_seq,
             """xpath_subs_seq""": xpath_subs_seq,
             """token_type_ids""": token_type_ids,
             """attention_mask""": input_mask,
         }
         return config, inputs_dict
  
  
  @require_torch
  class 		_SCREAMING_SNAKE_CASE    (	A       ,  A       ,  unittest.TestCase    ):
       __SCREAMING_SNAKE_CASE    				=						(
           (
               MarkupLMModel,
               MarkupLMForSequenceClassification,
               MarkupLMForTokenClassification,
               MarkupLMForQuestionAnswering,
           )
           if is_torch_available()
           else None
       )
       __SCREAMING_SNAKE_CASE    				=						(
           {
               '''feature-extraction''': MarkupLMModel,
               '''question-answering''': MarkupLMForQuestionAnswering,
               '''text-classification''': MarkupLMForSequenceClassification,
               '''token-classification''': MarkupLMForTokenClassification,
               '''zero-shot''': MarkupLMForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
  
  
       def 					__snake_case(     self       ,				A_       ,				A_       ,				A_       ,				A_       ,				A_						):
         # ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
         # (batch of pretokenized examples).
         return True
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Union[str, Any]       =    MarkupLMModelTester(self						)
         _UpperCAmelCase		:       str       =    ConfigTester(self       ,				config_class=A_       ,				hidden_size=37						)
  
  
       def 					__snake_case(     self						):
         self.config_tester.run_common_tests()
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       List[Any]       =    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*A_						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Optional[int]       =    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*A_						)
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Union[str, Any]       =    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*A_						)
  
  
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Any       =    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*A_						)
  
  
  
  
  def       a__						(		):
    _UpperCAmelCase		:       Any       =    """
      <!DOCTYPE html>
      <html>
      <head>
      <title>Page Title</title>
      </head>
      <body>
  
      <h1>This is a Heading</h1>
      <p>This is a paragraph.</p>
  
      </body>
      </html>
      """
  
    return html_string
  
  
  
  
  
  @require_torch
  class 		_SCREAMING_SNAKE_CASE    (	unittest.TestCase    ):
  
  
       @cached_property
       def 					__snake_case(     self						):
         # TODO use from_pretrained here
         _UpperCAmelCase		:       int       =    MarkupLMFeatureExtractor()
         _UpperCAmelCase		:       int       =    MarkupLMTokenizer.from_pretrained("""microsoft/markuplm-base"""						)
  
         return MarkupLMProcessor(A_       ,				A_						)
  
  
  
  
       @slow
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       List[Any]       =    MarkupLMModel.from_pretrained("""microsoft/markuplm-base"""						).to(A_						)
  
         _UpperCAmelCase		:       Optional[Any]       =    self.default_processor
  
         _UpperCAmelCase		:       str       =    processor(prepare_html_string()       ,				return_tensors="""pt"""						)
         _UpperCAmelCase		:       List[str]       =    inputs.to(A_						)
  
         # forward pass
         with torch.no_grad():
           _UpperCAmelCase		:       Optional[Any]       =    model(**A_						)
  
         # verify the last hidden states
         _UpperCAmelCase		:       Tuple       =    torch.Size([1, 14, 7_68]						)
         self.assertEqual(outputs.last_hidden_state.shape       ,				A_						)
  
         _UpperCAmelCase		:       List[Any]       =    torch.tensor(
             [[0.0_6_7_5, -0.0_0_5_2, 0.5_0_0_1], [-0.2_2_8_1, 0.0_8_0_2, 0.2_1_9_2], [-0.0_5_8_3, -0.3_3_1_1, 0.1_1_8_5]]						).to(A_						)
  
         self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]       ,				A_       ,				atol=1e-4						)						)
  
  
  
  
  
  