


'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      A__		:					Dict												=  None


A__		:					List[Any]												=  logging.get_logger(__name__)

A__		:					str												=  {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

A__		:					Union[str, Any]												=  {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

A__		:					Any												=  {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

A__		:					Dict												=  """▁"""

# Segments (not really needed)
A__		:					List[str]												=  0
A__		:					List[Any]												=  1
A__		:					Union[str, Any]												=  2
A__		:					List[Any]												=  3
A__		:					str												=  4





class    UpperCAmelCase_  (_UpperCAmelCase  ):





    """simple docstring"""






    lowerCamelCase  :					str							     =			VOCAB_FILES_NAMES
    lowerCamelCase  :					Dict							     =			PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase  :					Tuple							     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase  :					List[Any]							     =			'left'
    lowerCamelCase  :					Optional[Any]							     =			XLNetTokenizer







    def __init__(							self			,     SCREAMING_SNAKE_CASE_=None			,     SCREAMING_SNAKE_CASE_=None			,     SCREAMING_SNAKE_CASE_=False			,     SCREAMING_SNAKE_CASE_=True			,     SCREAMING_SNAKE_CASE_=False			,     SCREAMING_SNAKE_CASE_="<s>"			,     SCREAMING_SNAKE_CASE_="</s>"			,     SCREAMING_SNAKE_CASE_="<unk>"			,     SCREAMING_SNAKE_CASE_="<sep>"			,     SCREAMING_SNAKE_CASE_="<pad>"			,     SCREAMING_SNAKE_CASE_="<cls>"			,     SCREAMING_SNAKE_CASE_="<mask>"			,     SCREAMING_SNAKE_CASE_=["<eop>", "<eod>"]			,     **SCREAMING_SNAKE_CASE_			,     )				->     Union[str, Any]:
         # Mask token behave like a normal word, i.e. include the space before it
         __lowerCamelCase   :	str								=		AddedToken(SCREAMING_SNAKE_CASE_			,     lstrip=SCREAMING_SNAKE_CASE_			,     rstrip=SCREAMING_SNAKE_CASE_	) if isinstance(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	) else mask_token

         super().__init__(
             vocab_file=SCREAMING_SNAKE_CASE_			,     tokenizer_file=SCREAMING_SNAKE_CASE_			,     do_lower_case=SCREAMING_SNAKE_CASE_			,     remove_space=SCREAMING_SNAKE_CASE_			,     keep_accents=SCREAMING_SNAKE_CASE_			,     bos_token=SCREAMING_SNAKE_CASE_			,     eos_token=SCREAMING_SNAKE_CASE_			,     unk_token=SCREAMING_SNAKE_CASE_			,     sep_token=SCREAMING_SNAKE_CASE_			,     pad_token=SCREAMING_SNAKE_CASE_			,     cls_token=SCREAMING_SNAKE_CASE_			,     mask_token=SCREAMING_SNAKE_CASE_			,     additional_special_tokens=SCREAMING_SNAKE_CASE_			,     **SCREAMING_SNAKE_CASE_			,     )

         __lowerCamelCase   :	Union[str, Any]								=		3
         __lowerCamelCase   :	int								=		do_lower_case
         __lowerCamelCase   :	Optional[Any]								=		remove_space
         __lowerCamelCase   :	int								=		keep_accents
         __lowerCamelCase   :	Any								=		vocab_file
         __lowerCamelCase   :	Any								=		False if not self.vocab_file else True







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     List[int]:
         __lowerCamelCase   :	Optional[Any]								=		[self.sep_token_id]
         __lowerCamelCase   :	Any								=		[self.cls_token_id]
         if token_ids_a is None:
              return token_ids_a + sep + cls
         return token_ids_a + sep + token_ids_a + sep + cls







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     List[int]:
         __lowerCamelCase   :	int								=		[self.sep_token_id]
         __lowerCamelCase   :	Dict								=		[2]

         if token_ids_a is None:
              return len(token_ids_a + sep	) * [0] + cls_segment_id
         return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_ = None	)				->     Tuple[str]:
         if not self.can_save_slow_tokenizer:
              raise ValueError(
                  'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                  'tokenizer.'	)

         if not os.path.isdir(SCREAMING_SNAKE_CASE_	):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
              return
         __lowerCamelCase   :	Tuple								=		os.path.join(
             SCREAMING_SNAKE_CASE_			,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE_	):
              copyfile(self.vocab_file			,     SCREAMING_SNAKE_CASE_	)

         return (out_vocab_file,)










'''simple docstring'''



from importlib import import_module

from .logging import get_logger


A__		:					str												=  get_logger(__name__)





class    UpperCAmelCase_  :





    """simple docstring"""






    def __init__(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_=None	)				->     List[Any]:
         __lowerCamelCase   :	List[str]								=		attrs or []
         if module is not None:
              for key in module.__dict__:
                   if key in attrs or not key.startswith('__'	):
                        setattr(self			,     SCREAMING_SNAKE_CASE_			,     getattr(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)	)
         __lowerCamelCase   :	Optional[int]								=		module._original_module if isinstance(SCREAMING_SNAKE_CASE_			,     _PatchedModuleObj	) else module







class    UpperCAmelCase_  :





    """simple docstring"""






    lowerCamelCase  :					Tuple							     =			[]







    def __init__(							self			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_=None	)				->     Union[str, Any]:
         __lowerCamelCase   :	Optional[int]								=		obj
         __lowerCamelCase   :	List[Any]								=		target
         __lowerCamelCase   :	Union[str, Any]								=		new
         __lowerCamelCase   :	Union[str, Any]								=		target.split('.'	)[0]
         __lowerCamelCase   :	Dict								=		{}
         __lowerCamelCase   :	Dict								=		attrs or []







    def __enter__(							self	)				->     Optional[int]:
         *__lowerCamelCase				,       __lowerCamelCase   :	int								=		self.target.split('.'	)

         # Patch modules:
         # it's used to patch attributes of submodules like "os.path.join";
         # in this case we need to patch "os" and "os.path"

         for i in range(len(SCREAMING_SNAKE_CASE_	)	):
              try:
                   __lowerCamelCase   :	Optional[int]								=		import_module('.'.join(submodules[: i + 1]	)	)
              except ModuleNotFoundError:
                   continue
              # We iterate over all the globals in self.obj in case we find "os" or "os.path"
              for attr in self.obj.__dir__():
                   __lowerCamelCase   :	List[Any]								=		getattr(self.obj			,     SCREAMING_SNAKE_CASE_	)
                   # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
                   # This allows to patch renamed modules like "from os import path as ospath".
                   if obj_attr is submodule or (
                       (isinstance(SCREAMING_SNAKE_CASE_			,     _PatchedModuleObj	) and obj_attr._original_module is submodule)
                   ):
                        __lowerCamelCase   :	Optional[Any]								=		obj_attr
                        # patch at top level
                        setattr(self.obj			,     SCREAMING_SNAKE_CASE_			,     _PatchedModuleObj(SCREAMING_SNAKE_CASE_			,     attrs=self.attrs	)	)
                        __lowerCamelCase   :	str								=		getattr(self.obj			,     SCREAMING_SNAKE_CASE_	)
                        # construct lower levels patches
                        for key in submodules[i + 1 :]:
                             setattr(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     _PatchedModuleObj(getattr(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)			,     attrs=self.attrs	)	)
                             __lowerCamelCase   :	List[Any]								=		getattr(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)
                        # finally set the target attribute
                        setattr(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     self.new	)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

         if submodules:  # if it's an attribute of a submodule like "os.path.join"
              try:
                   __lowerCamelCase   :	Union[str, Any]								=		getattr(import_module('.'.join(SCREAMING_SNAKE_CASE_	)	)			,     SCREAMING_SNAKE_CASE_	)
              except (AttributeError, ModuleNotFoundError):
                   return
              # We iterate over all the globals in self.obj in case we find "os.path.join"
              for attr in self.obj.__dir__():
                   # We don't check for the name of the global, but rather if its value *is* "os.path.join".
                   # This allows to patch renamed attributes like "from os.path import join as pjoin".
                   if getattr(self.obj			,     SCREAMING_SNAKE_CASE_	) is attr_value:
                        __lowerCamelCase   :	Optional[int]								=		getattr(self.obj			,     SCREAMING_SNAKE_CASE_	)
                        setattr(self.obj			,     SCREAMING_SNAKE_CASE_			,     self.new	)
         elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
              __lowerCamelCase   :	List[Any]								=		globals()['__builtins__'][target_attr]
              setattr(self.obj			,     SCREAMING_SNAKE_CASE_			,     self.new	)
         else:
              raise RuntimeError(f'Tried to patch attribute {target_attr} instead of a submodule.'	)







    def __exit__(							self			,     *SCREAMING_SNAKE_CASE_	)				->     Optional[Any]:
         for attr in list(self.original	):
              setattr(self.obj			,     SCREAMING_SNAKE_CASE_			,     self.original.pop(SCREAMING_SNAKE_CASE_	)	)







    def       lowercase_				(							self	)				->     Optional[int]:
         self.__enter__()
         self._active_patches.append(self	)







    def       lowercase_				(							self	)				->     str:
         try:
              self._active_patches.remove(self	)
         except ValueError:
              # If the patch hasn't been started this will fail
              return None

         return self.__exit__()






