code: 
  
  
  
  """simple docstring"""
  
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import normalizers
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_layoutlm import LayoutLMTokenizer
  
  
  _lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)
  
  _lowerCAmelCase       :      Any							=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
  
  _lowerCAmelCase       :      Optional[Any]							=							{
      "vocab_file": {
          "microsoft/layoutlm-base-uncased": (
              "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
          ),
          "microsoft/layoutlm-large-uncased": (
              "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
          ),
      },
      "tokenizer_file": {
          "microsoft/layoutlm-base-uncased": (
              "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
          ),
          "microsoft/layoutlm-large-uncased": (
              "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
          ),
      },
  }
  
  _lowerCAmelCase       :      Union[str, Any]							=							{
      "microsoft/layoutlm-base-uncased": 5_12,
      "microsoft/layoutlm-large-uncased": 5_12,
  }
  
  _lowerCAmelCase       :      List[Any]							=							{
      "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
      "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
  }
  
  
  
  
  
  
  class 			UpperCAmelCase_						(       _UpperCamelCase		):
        __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
        __SCREAMING_SNAKE_CASE		:     Tuple				  =				PRETRAINED_VOCAB_FILES_MAP
        __SCREAMING_SNAKE_CASE		:     Optional[int]				  =				PRETRAINED_INIT_CONFIGURATION
        __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __SCREAMING_SNAKE_CASE		:     Optional[Any]				  =				LayoutLMTokenizer
  
  
  
        def __init__(  self  :			Tuple  ,			A  :			List[Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Union[str, Any]=True  ,			A  :			Optional[int]="[UNK]"  ,			A  :			Any="[SEP]"  ,			A  :			List[Any]="[PAD]"  ,			A  :			List[str]="[CLS]"  ,			A  :			Optional[int]="[MASK]"  ,			A  :			Optional[int]=True  ,			A  :			Dict=None  ,			**A  :			Union[str, Any]  ,			):
             super().__init__(
                 A  ,			tokenizer_file=A  ,			do_lower_case=A  ,			unk_token=A  ,			sep_token=A  ,			pad_token=A  ,			cls_token=A  ,			mask_token=A  ,			tokenize_chinese_chars=A  ,			strip_accents=A  ,			**A  ,			)
  
             _UpperCAmelCase    :			Tuple              =					json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
             if (
                 normalizer_state.get("lowercase"  ,			A		) != do_lower_case
                 or normalizer_state.get("strip_accents"  ,			A		) != strip_accents
                 or normalizer_state.get("handle_chinese_chars"  ,			A		) != tokenize_chinese_chars
             ):
                  _UpperCAmelCase    :			Dict              =					getattr(A  ,			normalizer_state.pop("type"		)		)
                  _UpperCAmelCase    :			Union[str, Any]              =					do_lower_case
                  _UpperCAmelCase    :			int              =					strip_accents
                  _UpperCAmelCase    :			Optional[Any]              =					tokenize_chinese_chars
                  _UpperCAmelCase    :			List[str]              =					normalizer_class(**A		)
  
             _UpperCAmelCase    :			Tuple              =					do_lower_case
  
  
  
        def 		snake_case_      (  self  :			Optional[Any]  ,			A  :			Dict  ,			A  :			List[str]=None		):
             _UpperCAmelCase    :			List[str]              =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]
  
             if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]
  
             return output
  
  
  
        def 		snake_case_      (  self  :			Dict  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
             _UpperCAmelCase    :			Any              =					[self.sep_token_id]
             _UpperCAmelCase    :			List[str]              =					[self.cls_token_id]
             if token_ids_a is None:
                  return len(cls + token_ids_a + sep		) * [0]
             return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]
  
  
  
        def 		snake_case_      (  self  :			Optional[int]  ,			A  :			str  ,			A  :			Optional[str] = None		):
             _UpperCAmelCase    :			Dict              =					self._tokenizer.model.save(A  ,			name=A		)
             return tuple(A		)
  
style_context: 
  
  
  
  """simple docstring"""
  
  
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
   from .tokenization_xglm import XGLMTokenizer
  else:
   _lowerCAmelCase       :      Tuple							=							None
  
  
  _lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)
  
  _lowerCAmelCase       :      str							=							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}
  
  _lowerCAmelCase       :      Optional[Any]							=							{
      "vocab_file": {
          "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
      },
      "tokenizer_file": {
          "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
      },
  }
  
  _lowerCAmelCase       :      Union[str, Any]							=							{
      "facebook/xglm-564M": 20_48,
  }
  
  
  
  
  
  
  class 			UpperCAmelCase_						(       _UpperCamelCase		):
        __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
        __SCREAMING_SNAKE_CASE		:     List[str]				  =				PRETRAINED_VOCAB_FILES_MAP
        __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __SCREAMING_SNAKE_CASE		:     int				  =				['input_ids', 'attention_mask']
        __SCREAMING_SNAKE_CASE		:     str				  =				XGLMTokenizer
  
  
  
        def __init__(  self  :			List[Any]  ,			A  :			Union[str, Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Optional[Any]="<s>"  ,			A  :			Optional[int]="</s>"  ,			A  :			str="</s>"  ,			A  :			int="<s>"  ,			A  :			int="<unk>"  ,			A  :			List[str]="<pad>"  ,			**A  :			Tuple  ,			):
             # Compatibility with the original tokenizer
             _UpperCAmelCase    :			Union[str, Any]              =					7
             _UpperCAmelCase    :			Optional[int]              =					[f'<madeupword{i}>' for i in range(self.num_madeup_words		)]
  
             _UpperCAmelCase    :			Any              =					kwargs.get("additional_special_tokens"  ,			[]		)
             kwargs["additional_special_tokens"] += [
                 word for word in madeup_words if word not in kwargs["additional_special_tokens"]
             ]
  
             super().__init__(
                 A  ,			tokenizer_file=A  ,			bos_token=A  ,			eos_token=A  ,			sep_token=A  ,			cls_token=A  ,			unk_token=A  ,			pad_token=A  ,			**A  ,			)
  
             _UpperCAmelCase    :			Any              =					vocab_file
             _UpperCAmelCase    :			Union[str, Any]              =					False if not self.vocab_file else True
  
  
  
        def 		snake_case_      (  self  :			List[str]  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
  
             if token_ids_a is None:
                  return [self.sep_token_id] + token_ids_a
             _UpperCAmelCase    :			Tuple              =					[self.sep_token_id]
             return sep + token_ids_a + sep + sep + token_ids_a
  
  
  
        def 		snake_case_      (  self  :			str  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
             _UpperCAmelCase    :			Optional[int]              =					[self.sep_token_id]
  
             if token_ids_a is None:
                  return len(sep + token_ids_a		) * [0]
             return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]
  
  
  
        def 		snake_case_      (  self  :			Dict  ,			A  :			str  ,			A  :			Optional[str] = None		):
             if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                      "tokenizer."		)
  
             if not os.path.isdir(A		):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory.'		)
                  return
             _UpperCAmelCase    :			str              =					os.path.join(
                 A  ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
  
             if os.path.abspath(self.vocab_file		) != os.path.abspath(A		):
                  copyfile(self.vocab_file  ,			A		)
  
             return (out_vocab_file,)
  