


"""simple docstring"""




from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class  __snake_case      (lowerCamelCase     ):
							__a				   =					'''MarkupLMFeatureExtractor'''
							__a				   =					('''MarkupLMTokenizer''', '''MarkupLMTokenizerFast''')
							__a				   =					True




							def __call__(    self:						Optional[Any]    ,       A_:						List[Any]=None    ,       A_:						Union[str, Any]=None    ,       A_:						int=None    ,       A_:						Union[str, Any]=None    ,       A_:						str=None    ,       A_:						bool = True    ,       A_:						Union[bool, str, PaddingStrategy] = False    ,       A_:						Union[bool, str, TruncationStrategy] = None    ,       A_:						Optional[int] = None    ,       A_:						int = 0    ,       A_:						Optional[int] = None    ,       A_:						Optional[bool] = None    ,       A_:						Optional[bool] = None    ,       A_:						bool = False    ,       A_:						bool = False    ,       A_:						bool = False    ,       A_:						bool = False    ,       A_:						bool = True    ,       A_:						Optional[Union[str, TensorType]] = None    ,       **A_:						List[Any]    ,       ):
										# first, create nodes and xpaths
										if self.parse_html:
													if html_strings is None:
																raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""      )

													if nodes is not None or xpaths is not None or node_labels is not None:
																raise ValueError(
																    """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""      )

													__lowerCamelCase        =			self.feature_extractor(A_      )
													__lowerCamelCase        =			features["""nodes"""]
													__lowerCamelCase        =			features["""xpaths"""]
										else:
													if html_strings is not None:
																raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""      )
													if nodes is None or xpaths is None:
																raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""      )

        # # second, apply the tokenizer
										if questions is not None and self.parse_html:
													if isinstance(A_    ,       A_      ):
																__lowerCamelCase        =			[questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

										__lowerCamelCase        =			self.tokenizer(
										    text=questions if questions is not None else nodes    ,       text_pair=nodes if questions is not None else None    ,       xpaths=A_    ,       node_labels=A_    ,       add_special_tokens=A_    ,       padding=A_    ,       truncation=A_    ,       max_length=A_    ,       stride=A_    ,       pad_to_multiple_of=A_    ,       return_token_type_ids=A_    ,       return_attention_mask=A_    ,       return_overflowing_tokens=A_    ,       return_special_tokens_mask=A_    ,       return_offsets_mapping=A_    ,       return_length=A_    ,       verbose=A_    ,       return_tensors=A_    ,       **A_    ,       )

										return encoded_inputs




							def      __a      (    self:						int    ,       *A_:						str    ,       **A_:						Union[str, Any]      ):
										return self.tokenizer.batch_decode(*A_    ,       **A_      )




							def      __a      (    self:						Any    ,       *A_:						Optional[int]    ,       **A_:						Any      ):
										return self.tokenizer.decode(*A_    ,       **A_      )




							@property
							def      __a      (    self:						int      ):
										__lowerCamelCase        =			self.tokenizer.model_input_names
										return tokenizer_input_names








"""simple docstring"""




# Lint as: python3

import hashlib
from typing import Union





def 			a_   (  lowercase__   :Union[str, int, bytes]    ):
			if isinstance(lowercase__, lowercase__    ):
						# Data already in bytes, returns as it as
						return hash_data
			elif isinstance(lowercase__, lowercase__    ):
						# We keep the data as it as for it ot be later encoded to UTF-8
						# However replace `\\` with `/` for Windows compatibility
						__lowerCamelCase        =			hash_data.replace("""\\""", """/"""    )
			elif isinstance(lowercase__, lowercase__    ):
						__lowerCamelCase        =			str(lowercase__    )
			else:
						# If data is not of the required type, raise error
						raise InvalidKeyError(lowercase__    )

			return hash_data.encode("""utf-8"""    )




class  __snake_case      (lowerCamelCase     ):




							def __init__(    self:						Union[str, Any]    ,       A_:						Tuple      ):
										__lowerCamelCase        =			"""\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
										__lowerCamelCase        =			f'\nFound Key {hash_data} of type {type(A_      )}'
										__lowerCamelCase        =			"""\nKeys should be either str, int or bytes type"""
										super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'      )




class  __snake_case      (lowerCamelCase     ):




							def __init__(    self:						Any    ,       A_:						Dict    ,       A_:						List[Any]    ,       A_:						List[str]=""      ):
										__lowerCamelCase        =			key
										__lowerCamelCase        =			duplicate_key_indices
										__lowerCamelCase        =			fix_msg
										__lowerCamelCase        =			"""Found multiple examples generated with the same key"""
										if len(A_      ) <= 20:
													__lowerCamelCase        =			f'\nThe examples at index {", ".join(A_      )} have the key {key}'
										else:
													__lowerCamelCase        =			f'\nThe examples at index {", ".join(duplicate_key_indices[:20]      )}... ({len(A_      ) - 20} more) have the key {key}'
										__lowerCamelCase        =			"""\n""" + fix_msg if fix_msg else """"""
										super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'      )




class  __snake_case      :




							def __init__(    self:						Dict    ,       A_:						str      ):
										__lowerCamelCase        =			hashlib.mda(_as_bytes(A_      )      )




							def      __a      (    self:						int    ,       A_:						Union[str, int, bytes]      ):
										__lowerCamelCase        =			self._split_mda.copy()
										__lowerCamelCase        =			_as_bytes(A_      )
										mda.update(A_      )
										# Convert to integer with hexadecimal conversion
										return int(mda.hexdigest()    ,       16      )




