


"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


lowercase_							 =       logging.get_logger(__name__)

lowercase_							 =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase_							 =       {
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

lowercase_							 =       {
    'microsoft/layoutlm-base-uncased': 5_12,
    'microsoft/layoutlm-large-uncased': 5_12,
}

lowercase_							 =       {
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}







class       __a	(   __snake_case       ):
				lowerCamelCase    :					Union[str, Any] 					=VOCAB_FILES_NAMES
				lowerCamelCase    :					str 					=PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase    :					Any 					=PRETRAINED_INIT_CONFIGURATION
				lowerCamelCase    :					Union[str, Any] 					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase    :					Optional[Any] 					=LayoutLMTokenizer







				def __init__(   self			,  UpperCAmelCase=None			,  UpperCAmelCase=None			,  UpperCAmelCase=True			,  UpperCAmelCase="[UNK]"			,  UpperCAmelCase="[SEP]"			,  UpperCAmelCase="[PAD]"			,  UpperCAmelCase="[CLS]"			,  UpperCAmelCase="[MASK]"			,  UpperCAmelCase=True			,  UpperCAmelCase=None			,  **UpperCAmelCase			,  ):

					'''simple docstring'''






					super().__init__(
					    UpperCAmelCase			,  tokenizer_file=UpperCAmelCase			,  do_lower_case=UpperCAmelCase			,  unk_token=UpperCAmelCase			,  sep_token=UpperCAmelCase			,  pad_token=UpperCAmelCase			,  cls_token=UpperCAmelCase			,  mask_token=UpperCAmelCase			,  tokenize_chinese_chars=UpperCAmelCase			,  strip_accents=UpperCAmelCase			,  **UpperCAmelCase			,  )

					lowerCAmelCase_       = json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
					if (
					    normalizer_state.get('''lowercase'''			,  UpperCAmelCase      ) != do_lower_case
					    or normalizer_state.get('''strip_accents'''			,  UpperCAmelCase      ) != strip_accents
					    or normalizer_state.get('''handle_chinese_chars'''			,  UpperCAmelCase      ) != tokenize_chinese_chars
					):
						lowerCAmelCase_       = getattr(UpperCAmelCase			,  normalizer_state.pop('''type'''      )      )
						lowerCAmelCase_       = do_lower_case
						lowerCAmelCase_       = strip_accents
						lowerCAmelCase_       = tokenize_chinese_chars
						lowerCAmelCase_       = normalizer_class(**UpperCAmelCase      )

					lowerCAmelCase_       = do_lower_case







				def     lowerCamelCase_    (   self			,  UpperCAmelCase			,  UpperCAmelCase=None      ):

					'''simple docstring'''






					lowerCAmelCase_       = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output







				def     lowerCamelCase_    (   self			,  UpperCAmelCase			,  UpperCAmelCase = None      ):

					'''simple docstring'''






					lowerCAmelCase_       = [self.sep_token_id]
					lowerCAmelCase_       = [self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep      ) * [0]
					return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







				def     lowerCamelCase_    (   self			,  UpperCAmelCase			,  UpperCAmelCase = None      ):

					'''simple docstring'''






					lowerCAmelCase_       = self._tokenizer.model.save(UpperCAmelCase			,  name=UpperCAmelCase      )
					return tuple(UpperCAmelCase      )



"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase_							 =       {
    'configuration_prophetnet': ['PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ProphetNetConfig'],
    'tokenization_prophetnet': ['ProphetNetTokenizer'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase_							 =       [
		    'PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'ProphetNetDecoder',
		    'ProphetNetEncoder',
		    'ProphetNetForCausalLM',
		    'ProphetNetForConditionalGeneration',
		    'ProphetNetModel',
		    'ProphetNetPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
		from .tokenization_prophetnet import ProphetNetTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_prophetnet import (
				    PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ProphetNetDecoder,
				    ProphetNetEncoder,
				    ProphetNetForCausalLM,
				    ProphetNetForConditionalGeneration,
				    ProphetNetModel,
				    ProphetNetPreTrainedModel,
				)

else:
		import sys

		lowercase_							 =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)