



import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class       __lowerCAmelCase				(  lowerCamelCase__ ,		lowerCamelCase__ ,		lowerCamelCase__ ):




				@register_to_config
				def __init__(							self		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case		,       _snake_case = False		,       ):



									"""simple docstring"""





									super().__init__()

									_lowerCAmelCase      			=					nn.Linear(_snake_case		,       _snake_case		,       bias=_snake_case	)

									_lowerCAmelCase      			=					nn.Embedding(_snake_case		,       _snake_case	)
									_lowerCAmelCase      			=					False

									_lowerCAmelCase      			=					nn.Dropout(p=_snake_case	)

									_lowerCAmelCase      			=					TaConfig(
									    d_model=_snake_case		,       num_heads=_snake_case		,       d_kv=_snake_case		,       d_ff=_snake_case		,       feed_forward_proj=_snake_case		,       dropout_rate=_snake_case		,       is_decoder=_snake_case		,       is_encoder_decoder=_snake_case		,       )
									_lowerCAmelCase      			=					nn.ModuleList()
									for lyr_num in range(_snake_case	):
														_lowerCAmelCase      			=					TaBlock(_snake_case	)
														self.encoders.append(_snake_case	)

									_lowerCAmelCase      			=					TaLayerNorm(_snake_case	)
									_lowerCAmelCase      			=					nn.Dropout(p=_snake_case	)




				def 							snake_case     (							self		,       _snake_case		,       _snake_case	):



									"""simple docstring"""





									_lowerCAmelCase      			=					self.input_proj(_snake_case	)

									# terminal relative positional encodings
									_lowerCAmelCase      			=					encoder_inputs.shape[1]
									_lowerCAmelCase      			=					torch.arange(_snake_case		,       device=encoder_inputs.device	)

									_lowerCAmelCase      			=					encoder_inputs_mask.sum(-1	)
									_lowerCAmelCase      			=					torch.roll(input_positions.unsqueeze(0	)		,       tuple(seq_lens.tolist()	)		,       dims=0	)
									x += self.position_encoding(_snake_case	)

									_lowerCAmelCase      			=					self.dropout_pre(_snake_case	)

									# inverted the attention mask
									_lowerCAmelCase      			=					encoder_inputs.size()
									_lowerCAmelCase      			=					self.get_extended_attention_mask(_snake_case		,       _snake_case	)

									for lyr in self.encoders:
														_lowerCAmelCase      			=					lyr(_snake_case		,       _snake_case	)[0]
									_lowerCAmelCase      			=					self.layer_norm(_snake_case	)

									return self.dropout_post(_snake_case	), encoder_inputs_mask








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


A__			   = {
    """configuration_videomae""": ["""VIDEOMAE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """VideoMAEConfig"""],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A__			   = [
	    """VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """VideoMAEForPreTraining""",
	    """VideoMAEModel""",
	    """VideoMAEPreTrainedModel""",
	    """VideoMAEForVideoClassification""",
	]

try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A__			   = ["""VideoMAEFeatureExtractor"""]
	A__			   = ["""VideoMAEImageProcessor"""]

if TYPE_CHECKING:
	from .configuration_videomae import VIDEOMAE_PRETRAINED_CONFIG_ARCHIVE_MAP, VideoMAEConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_videomae import (
		    VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST,
		    VideoMAEForPreTraining,
		    VideoMAEForVideoClassification,
		    VideoMAEModel,
		    VideoMAEPreTrainedModel,
		)

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_videomae import VideoMAEFeatureExtractor
		from .image_processing_videomae import VideoMAEImageProcessor

else:
	import sys

	A__			   = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



