



"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


_lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)

_lowerCAmelCase       :      Any							=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase       :      Optional[Any]							=							{
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCAmelCase       :      Union[str, Any]							=							{
    "microsoft/layoutlm-base-uncased": 5_12,
    "microsoft/layoutlm-large-uncased": 5_12,
}

_lowerCAmelCase       :      List[Any]							=							{
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}






class 			UpperCAmelCase_						(       _UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:     Tuple				  =				PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:     Optional[int]				  =				PRETRAINED_INIT_CONFIGURATION
      __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:     Optional[Any]				  =				LayoutLMTokenizer



      def __init__(  self  :			Tuple  ,			A  :			List[Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Union[str, Any]=True  ,			A  :			Optional[int]="[UNK]"  ,			A  :			Any="[SEP]"  ,			A  :			List[Any]="[PAD]"  ,			A  :			List[str]="[CLS]"  ,			A  :			Optional[int]="[MASK]"  ,			A  :			Optional[int]=True  ,			A  :			Dict=None  ,			**A  :			Union[str, Any]  ,			):
           super().__init__(
               A  ,			tokenizer_file=A  ,			do_lower_case=A  ,			unk_token=A  ,			sep_token=A  ,			pad_token=A  ,			cls_token=A  ,			mask_token=A  ,			tokenize_chinese_chars=A  ,			strip_accents=A  ,			**A  ,			)

           _UpperCAmelCase    :			Tuple              =					json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
           if (
               normalizer_state.get("lowercase"  ,			A		) != do_lower_case
               or normalizer_state.get("strip_accents"  ,			A		) != strip_accents
               or normalizer_state.get("handle_chinese_chars"  ,			A		) != tokenize_chinese_chars
           ):
                _UpperCAmelCase    :			Dict              =					getattr(A  ,			normalizer_state.pop("type"		)		)
                _UpperCAmelCase    :			Union[str, Any]              =					do_lower_case
                _UpperCAmelCase    :			int              =					strip_accents
                _UpperCAmelCase    :			Optional[Any]              =					tokenize_chinese_chars
                _UpperCAmelCase    :			List[str]              =					normalizer_class(**A		)

           _UpperCAmelCase    :			Tuple              =					do_lower_case



      def 		snake_case_      (  self  :			Optional[Any]  ,			A  :			Dict  ,			A  :			List[str]=None		):
           _UpperCAmelCase    :			List[str]              =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

           return output



      def 		snake_case_      (  self  :			Dict  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
           _UpperCAmelCase    :			Any              =					[self.sep_token_id]
           _UpperCAmelCase    :			List[str]              =					[self.cls_token_id]
           if token_ids_a is None:
                return len(cls + token_ids_a + sep		) * [0]
           return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]



      def 		snake_case_      (  self  :			Optional[int]  ,			A  :			str  ,			A  :			Optional[str] = None		):
           _UpperCAmelCase    :			Dict              =					self._tokenizer.model.save(A  ,			name=A		)
           return tuple(A		)





"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_xglm import XGLMTokenizer
else:
 _lowerCAmelCase       :      Tuple							=							None


_lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)

_lowerCAmelCase       :      str							=							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase       :      Optional[Any]							=							{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
    },
}

_lowerCAmelCase       :      Union[str, Any]							=							{
    "facebook/xglm-564M": 20_48,
}






class 			UpperCAmelCase_						(       _UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:     List[str]				  =				PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:     int				  =				['input_ids', 'attention_mask']
      __SCREAMING_SNAKE_CASE		:     str				  =				XGLMTokenizer



      def __init__(  self  :			List[Any]  ,			A  :			Union[str, Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Optional[Any]="<s>"  ,			A  :			Optional[int]="</s>"  ,			A  :			str="</s>"  ,			A  :			int="<s>"  ,			A  :			int="<unk>"  ,			A  :			List[str]="<pad>"  ,			**A  :			Tuple  ,			):
           # Compatibility with the original tokenizer
           _UpperCAmelCase    :			Union[str, Any]              =					7
           _UpperCAmelCase    :			Optional[int]              =					[f'<madeupword{i}>' for i in range(self.num_madeup_words		)]

           _UpperCAmelCase    :			Any              =					kwargs.get("additional_special_tokens"  ,			[]		)
           kwargs["additional_special_tokens"] += [
               word for word in madeup_words if word not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               A  ,			tokenizer_file=A  ,			bos_token=A  ,			eos_token=A  ,			sep_token=A  ,			cls_token=A  ,			unk_token=A  ,			pad_token=A  ,			**A  ,			)

           _UpperCAmelCase    :			Any              =					vocab_file
           _UpperCAmelCase    :			Union[str, Any]              =					False if not self.vocab_file else True



      def 		snake_case_      (  self  :			List[str]  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):

           if token_ids_a is None:
                return [self.sep_token_id] + token_ids_a
           _UpperCAmelCase    :			Tuple              =					[self.sep_token_id]
           return sep + token_ids_a + sep + sep + token_ids_a



      def 		snake_case_      (  self  :			str  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
           _UpperCAmelCase    :			Optional[int]              =					[self.sep_token_id]

           if token_ids_a is None:
                return len(sep + token_ids_a		) * [0]
           return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]



      def 		snake_case_      (  self  :			Dict  ,			A  :			str  ,			A  :			Optional[str] = None		):
           if not self.can_save_slow_tokenizer:
                raise ValueError(
                    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                    "tokenizer."		)

           if not os.path.isdir(A		):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory.'		)
                return
           _UpperCAmelCase    :			str              =					os.path.join(
               A  ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

           if os.path.abspath(self.vocab_file		) != os.path.abspath(A		):
                copyfile(self.vocab_file  ,			A		)

           return (out_vocab_file,)
