






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A										=      logging.get_logger(__name__)

A										=      {
    'facebook/hubert-base-ls960': 'https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}


class 				__a				(	__A    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Optional[Any]					     =    """hubert"""


		def __init__(    self			,     UpperCamelCase__=32			,     UpperCamelCase__=768			,     UpperCamelCase__=12			,     UpperCamelCase__=12			,     UpperCamelCase__=3072			,     UpperCamelCase__="gelu"			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=True			,     UpperCamelCase__=0.0			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.02			,     UpperCamelCase__=1E-5			,     UpperCamelCase__="group"			,     UpperCamelCase__="gelu"			,     UpperCamelCase__=(512, 512, 512, 512, 512, 512, 512)			,     UpperCamelCase__=(5, 2, 2, 2, 2, 2, 2)			,     UpperCamelCase__=(10, 3, 3, 3, 3, 2, 2)			,     UpperCamelCase__=False			,     UpperCamelCase__=128			,     UpperCamelCase__=16			,     UpperCamelCase__=False			,     UpperCamelCase__=True			,     UpperCamelCase__=0.05			,     UpperCamelCase__=10			,     UpperCamelCase__=2			,     UpperCamelCase__=0.0			,     UpperCamelCase__=10			,     UpperCamelCase__=0			,     UpperCamelCase__="sum"			,     UpperCamelCase__=False			,     UpperCamelCase__=False			,     UpperCamelCase__=256			,     UpperCamelCase__=0			,     UpperCamelCase__=1			,     UpperCamelCase__=2			,     **UpperCamelCase__			,     ):
						super().__init__(**UpperCamelCase__			,     pad_token_id=UpperCamelCase__			,     bos_token_id=UpperCamelCase__			,     eos_token_id=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						int    				= hidden_size
						SCREAMING_SNAKE_CASE_ :						List[str]    				= feat_extract_norm
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= feat_extract_activation
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= list(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Tuple    				= list(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= list(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						str    				= conv_bias
						SCREAMING_SNAKE_CASE_ :						int    				= num_conv_pos_embeddings
						SCREAMING_SNAKE_CASE_ :						int    				= num_conv_pos_embedding_groups
						SCREAMING_SNAKE_CASE_ :						int    				= len(self.conv_dim							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= num_hidden_layers
						SCREAMING_SNAKE_CASE_ :						List[str]    				= intermediate_size
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= hidden_act
						SCREAMING_SNAKE_CASE_ :						Dict    				= num_attention_heads
						SCREAMING_SNAKE_CASE_ :						List[str]    				= hidden_dropout
						SCREAMING_SNAKE_CASE_ :						List[str]    				= attention_dropout
						SCREAMING_SNAKE_CASE_ :						Any    				= activation_dropout
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= feat_proj_layer_norm
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= feat_proj_dropout
						SCREAMING_SNAKE_CASE_ :						Any    				= final_dropout
						SCREAMING_SNAKE_CASE_ :						Tuple    				= layerdrop
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= layer_norm_eps
						SCREAMING_SNAKE_CASE_ :						Dict    				= initializer_range
						SCREAMING_SNAKE_CASE_ :						str    				= vocab_size
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= do_stable_layer_norm
						SCREAMING_SNAKE_CASE_ :						int    				= use_weighted_layer_sum
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= classifier_proj_size

						if (
						    (len(self.conv_stride							) != self.num_feat_extract_layers)
						    or (len(self.conv_kernel							) != self.num_feat_extract_layers)
						    or (len(self.conv_dim							) != self.num_feat_extract_layers)
						):
										raise ValueError(
										    'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
										    ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
										    F''' {len(self.conv_dim							)}`, `len(config.conv_stride) = {len(self.conv_stride							)}`,'''
										    F''' `len(config.conv_kernel) = {len(self.conv_kernel							)}`.'''							)

						# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
						SCREAMING_SNAKE_CASE_ :						List[str]    				= apply_spec_augment
						SCREAMING_SNAKE_CASE_ :						Dict    				= mask_time_prob
						SCREAMING_SNAKE_CASE_ :						int    				= mask_time_length
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= mask_time_min_masks
						SCREAMING_SNAKE_CASE_ :						Dict    				= mask_feature_prob
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= mask_feature_length
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= mask_feature_min_masks

						# ctc loss
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= ctc_loss_reduction
						SCREAMING_SNAKE_CASE_ :						int    				= ctc_zero_infinity






		@property
		def 	__snake_case      (    self							):
						return functools.reduce(operator.mul			,     self.conv_stride			,     1							)







import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MarkupLMForQuestionAnswering,
				    MarkupLMForSequenceClassification,
				    MarkupLMForTokenClassification,
				    MarkupLMModel,
				)

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer


class 				__a				:





		'''simple docstring'''



		def __init__(    self			,     UpperCamelCase__			,     UpperCamelCase__=13			,     UpperCamelCase__=7			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=99			,     UpperCamelCase__=32			,     UpperCamelCase__=5			,     UpperCamelCase__=4			,     UpperCamelCase__=37			,     UpperCamelCase__="gelu"			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=512			,     UpperCamelCase__=16			,     UpperCamelCase__=2			,     UpperCamelCase__=0.02			,     UpperCamelCase__=3			,     UpperCamelCase__=None			,     UpperCamelCase__=20			,     UpperCamelCase__=30			,     UpperCamelCase__=2			,     UpperCamelCase__=2			,     UpperCamelCase__=10			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= parent
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= batch_size
						SCREAMING_SNAKE_CASE_ :						int    				= seq_length
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= is_training
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_input_mask
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_token_type_ids
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= use_labels
						SCREAMING_SNAKE_CASE_ :						int    				= vocab_size
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= hidden_size
						SCREAMING_SNAKE_CASE_ :						Any    				= num_hidden_layers
						SCREAMING_SNAKE_CASE_ :						int    				= num_attention_heads
						SCREAMING_SNAKE_CASE_ :						Any    				= intermediate_size
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= hidden_act
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= hidden_dropout_prob
						SCREAMING_SNAKE_CASE_ :						List[str]    				= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= max_position_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= type_vocab_size
						SCREAMING_SNAKE_CASE_ :						str    				= type_sequence_label_size
						SCREAMING_SNAKE_CASE_ :						Dict    				= initializer_range
						SCREAMING_SNAKE_CASE_ :						int    				= num_labels
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= scope
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= max_xpath_tag_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_xpath_subs_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Dict    				= tag_pad_id
						SCREAMING_SNAKE_CASE_ :						str    				= subs_pad_id
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_depth


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_tag_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_subs_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Any    				= None
						if self.use_input_mask:
										SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     vocab_size=2							)

						SCREAMING_SNAKE_CASE_ :						Dict    				= None
						if self.use_token_type_ids:
										SCREAMING_SNAKE_CASE_ :						int    				= ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size							)

						SCREAMING_SNAKE_CASE_ :						int    				= None
						SCREAMING_SNAKE_CASE_ :						Tuple    				= None
						if self.use_labels:
										SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor([self.batch_size]			,     self.type_sequence_label_size							)
										SCREAMING_SNAKE_CASE_ :						Any    				= ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.get_config()

						return (
						    config,
						    input_ids,
						    xpath_tags_seq,
						    xpath_subs_seq,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						)


		def 	__snake_case      (    self							):
						return MarkupLMConfig(
						    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     initializer_range=self.initializer_range			,     max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings			,     max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings			,     tag_pad_id=self.tag_pad_id			,     subs_pad_id=self.subs_pad_id			,     max_depth=self.max_depth			,     )


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMModel(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						print('Configs:'			,     model.config.tag_pad_id			,     model.config.subs_pad_id							)
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= model(UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						List[str]    				= model(UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						int    				= model(UpperCamelCase__							)
						self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)							)
						self.parent.assertEqual(result.pooler_output.shape			,     (self.batch_size, self.hidden_size)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Dict    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMForSequenceClassification(UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMForTokenClassification(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= MarkupLMForQuestionAnswering(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						int    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     start_positions=UpperCamelCase__			,     end_positions=UpperCamelCase__			,     )
						self.parent.assertEqual(result.start_logits.shape			,     (self.batch_size, self.seq_length)							)
						self.parent.assertEqual(result.end_logits.shape			,     (self.batch_size, self.seq_length)							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							
						) :						int    				= config_and_inputs
						SCREAMING_SNAKE_CASE_ :						Any    				= {
						    'input_ids': input_ids,
						    'xpath_tags_seq': xpath_tags_seq,
						    'xpath_subs_seq': xpath_subs_seq,
						    'token_type_ids': token_type_ids,
						    'attention_mask': input_mask,
						}
						return config, inputs_dict






@require_torch
class 				__a				(	__A			,						__A			,						unittest.TestCase    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Dict					     =    (
		    (
		        MarkupLMModel,
		        MarkupLMForSequenceClassification,
		        MarkupLMForTokenClassification,
		        MarkupLMForQuestionAnswering,
		    )
		    if is_torch_available()
		    else None
		)
		UpperCAmelCase__   :					Union[str, Any]					     =    (
		    {
		        """feature-extraction""": MarkupLMModel,
		        """question-answering""": MarkupLMForQuestionAnswering,
		        """text-classification""": MarkupLMForSequenceClassification,
		        """token-classification""": MarkupLMForTokenClassification,
		        """zero-shot""": MarkupLMForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__							):
						# ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
						# (batch of pretokenized examples).
						return True


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						int    				= MarkupLMModelTester(self							)
						SCREAMING_SNAKE_CASE_ :						Any    				= ConfigTester(self			,     config_class=UpperCamelCase__			,     hidden_size=37							)


		def 	__snake_case      (    self							):
						self.config_tester.run_common_tests()


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__							)






def        _lowerCamelCase(      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_ :						int    				= '\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    '

				return html_string


@require_torch
class 				__a				(	unittest.TestCase    ):





		'''simple docstring'''



		@cached_property
		def 	__snake_case      (    self							):
						# TODO use from_pretrained here
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMFeatureExtractor()
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMTokenizer.from_pretrained('microsoft/markuplm-base'							)

						return MarkupLMProcessor(UpperCamelCase__			,     UpperCamelCase__							)






		@slow
		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Tuple    				= MarkupLMModel.from_pretrained('microsoft/markuplm-base'							).to(UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.default_processor

						SCREAMING_SNAKE_CASE_ :						int    				= processor(prepare_html_string()			,     return_tensors='pt'							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= inputs.to(UpperCamelCase__							)

						# forward pass
						with torch.no_grad():
										SCREAMING_SNAKE_CASE_ :						str    				= model(**UpperCamelCase__							)

						# verify the last hidden states
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= torch.Size([1, 14, 768]							)
						self.assertEqual(outputs.last_hidden_state.shape			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= torch.tensor(
						    [[0.06_75, -0.00_52, 0.50_01], [-0.22_81, 0.08_02, 0.21_92], [-0.05_83, -0.33_11, 0.11_85]]							).to(UpperCamelCase__							)

						self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,     UpperCamelCase__			,     atol=1E-4							)							)