


'''simple docstring'''



from math import loga



def 			UpperCAmelCase__					(	UpperCAmelCase_    :				int							)     ->		int:
     if a < 0:
          raise ValueError('Input value must be a positive integer'							)
     elif isinstance(UpperCAmelCase_      ,    UpperCAmelCase_							):
          raise TypeError('Input value must be a \'int\' type'							)
     return 0 if (a == 0) else int(loga(a & -a							)							)


if __name__ == "__main__":
      import doctest

      doctest.testmod()










'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      A__		:					Dict												=  None


A__		:					List[Any]												=  logging.get_logger(__name__)

A__		:					str												=  {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

A__		:					Union[str, Any]												=  {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

A__		:					Any												=  {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

A__		:					Dict												=  """▁"""

# Segments (not really needed)
A__		:					List[str]												=  0
A__		:					List[Any]												=  1
A__		:					Union[str, Any]												=  2
A__		:					List[Any]												=  3
A__		:					str												=  4





class    UpperCAmelCase_  (_UpperCAmelCase  ):





    """simple docstring"""






    lowerCamelCase  :					str							     =			VOCAB_FILES_NAMES
    lowerCamelCase  :					Dict							     =			PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase  :					Tuple							     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase  :					List[Any]							     =			'left'
    lowerCamelCase  :					Optional[Any]							     =			XLNetTokenizer







    def __init__(							self			,     SCREAMING_SNAKE_CASE_=None			,     SCREAMING_SNAKE_CASE_=None			,     SCREAMING_SNAKE_CASE_=False			,     SCREAMING_SNAKE_CASE_=True			,     SCREAMING_SNAKE_CASE_=False			,     SCREAMING_SNAKE_CASE_="<s>"			,     SCREAMING_SNAKE_CASE_="</s>"			,     SCREAMING_SNAKE_CASE_="<unk>"			,     SCREAMING_SNAKE_CASE_="<sep>"			,     SCREAMING_SNAKE_CASE_="<pad>"			,     SCREAMING_SNAKE_CASE_="<cls>"			,     SCREAMING_SNAKE_CASE_="<mask>"			,     SCREAMING_SNAKE_CASE_=["<eop>", "<eod>"]			,     **SCREAMING_SNAKE_CASE_			,     )				->     Union[str, Any]:
         # Mask token behave like a normal word, i.e. include the space before it
         __lowerCamelCase   :	str								=		AddedToken(SCREAMING_SNAKE_CASE_			,     lstrip=SCREAMING_SNAKE_CASE_			,     rstrip=SCREAMING_SNAKE_CASE_	) if isinstance(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	) else mask_token

         super().__init__(
             vocab_file=SCREAMING_SNAKE_CASE_			,     tokenizer_file=SCREAMING_SNAKE_CASE_			,     do_lower_case=SCREAMING_SNAKE_CASE_			,     remove_space=SCREAMING_SNAKE_CASE_			,     keep_accents=SCREAMING_SNAKE_CASE_			,     bos_token=SCREAMING_SNAKE_CASE_			,     eos_token=SCREAMING_SNAKE_CASE_			,     unk_token=SCREAMING_SNAKE_CASE_			,     sep_token=SCREAMING_SNAKE_CASE_			,     pad_token=SCREAMING_SNAKE_CASE_			,     cls_token=SCREAMING_SNAKE_CASE_			,     mask_token=SCREAMING_SNAKE_CASE_			,     additional_special_tokens=SCREAMING_SNAKE_CASE_			,     **SCREAMING_SNAKE_CASE_			,     )

         __lowerCamelCase   :	Union[str, Any]								=		3
         __lowerCamelCase   :	int								=		do_lower_case
         __lowerCamelCase   :	Optional[Any]								=		remove_space
         __lowerCamelCase   :	int								=		keep_accents
         __lowerCamelCase   :	Any								=		vocab_file
         __lowerCamelCase   :	Any								=		False if not self.vocab_file else True







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     List[int]:
         __lowerCamelCase   :	Optional[Any]								=		[self.sep_token_id]
         __lowerCamelCase   :	Any								=		[self.cls_token_id]
         if token_ids_a is None:
              return token_ids_a + sep + cls
         return token_ids_a + sep + token_ids_a + sep + cls







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     List[int]:
         __lowerCamelCase   :	int								=		[self.sep_token_id]
         __lowerCamelCase   :	Dict								=		[2]

         if token_ids_a is None:
              return len(token_ids_a + sep	) * [0] + cls_segment_id
         return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     Tuple[str]:
         if not self.can_save_slow_tokenizer:
              raise ValueError(
                  'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                  'tokenizer.'	)

         if not os.path.isdir(SCREAMING_SNAKE_CASE_	):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
              return
         __lowerCamelCase   :	Tuple								=		os.path.join(
             SCREAMING_SNAKE_CASE_			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE_	):
              copyfile(self.vocab_file			,     SCREAMING_SNAKE_CASE_	)

         return (out_vocab_file,)






