






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__UpperCAmelCase      =				{
    "configuration_markuplm": ["MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP", "MarkupLMConfig"],
    "feature_extraction_markuplm": ["MarkupLMFeatureExtractor"],
    "processing_markuplm": ["MarkupLMProcessor"],
    "tokenization_markuplm": ["MarkupLMTokenizer"],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      =				["MarkupLMTokenizerFast"]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      =				[
          "MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST",
          "MarkupLMForQuestionAnswering",
          "MarkupLMForSequenceClassification",
          "MarkupLMForTokenClassification",
          "MarkupLMModel",
          "MarkupLMPreTrainedModel",
      ]


if TYPE_CHECKING:
      from .configuration_markuplm import MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP, MarkupLMConfig
      from .feature_extraction_markuplm import MarkupLMFeatureExtractor
      from .processing_markuplm import MarkupLMProcessor
      from .tokenization_markuplm import MarkupLMTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_markuplm_fast import MarkupLMTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_markuplm import (
                MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST,
                MarkupLMForQuestionAnswering,
                MarkupLMForSequenceClassification,
                MarkupLMForTokenClassification,
                MarkupLMModel,
                MarkupLMPreTrainedModel,
            )


else:
      import sys

      __UpperCAmelCase      =				_LazyModule(__name__, globals()["__file__"], _import_structure)








"""simple docstring"""







import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class  __A							(			_SCREAMING_SNAKE_CASE,       _SCREAMING_SNAKE_CASE,       _SCREAMING_SNAKE_CASE     ):






 """simple docstring"""




 @register_to_config
 def __init__(						self					,       __A					,       __A					,       __A					,       __A					,       __A					,       __A					,       __A					,       __A					,       __A					,       __A = False					,       )   ->	Optional[Any]:
     super().__init__()

     a											=nn.Linear(__A					,       __A					,       bias=__A		)

     a											=nn.Embedding(__A					,       __A		)
     a											=False

     a											=nn.Dropout(p=__A		)

     a											=TaConfig(
         d_model=__A					,       num_heads=__A					,       d_kv=__A					,       d_ff=__A					,       feed_forward_proj=__A					,       dropout_rate=__A					,       is_decoder=__A					,       is_encoder_decoder=__A					,       )
     a											=nn.ModuleList()
     for lyr_num in range(__A		):
         a											=TaBlock(__A		)
         self.encoders.append(__A		)

     a											=TaLayerNorm(__A		)
     a											=nn.Dropout(p=__A		)





 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A		)   ->	str:
     a											=self.input_proj(__A		)

     # terminal relative positional encodings
     a											=encoder_inputs.shape[1]
     a											=torch.arange(__A					,       device=encoder_inputs.device		)

     a											=encoder_inputs_mask.sum(-1		)
     a											=torch.roll(input_positions.unsqueeze(0		)					,       tuple(seq_lens.tolist()		)					,       dims=0		)
     x += self.position_encoding(__A		)

     a											=self.dropout_pre(__A		)

     # inverted the attention mask
     a											=encoder_inputs.size()
     a											=self.get_extended_attention_mask(__A					,       __A		)

     for lyr in self.encoders:
         a											=lyr(__A					,       __A		)[0]
     a											=self.layer_norm(__A		)

     return self.dropout_post(__A		), encoder_inputs_mask