


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__lowerCAmelCase		       =							logging.get_logger(__name__)

__lowerCAmelCase		       =							{'''vocab_file''': '''spiece.model'''}

__lowerCAmelCase		       =							{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

__lowerCAmelCase		       =							{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
__lowerCAmelCase		       =							0
__lowerCAmelCase		       =							1
__lowerCAmelCase		       =							2
__lowerCAmelCase		       =							3
__lowerCAmelCase		       =							4





class   __a			(    __UpperCamelCase      ):
 __lowercase :							Any        = VOCAB_FILES_NAMES
 __lowercase :							str        = PRETRAINED_VOCAB_FILES_MAP
 __lowercase :							Dict        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase :							Union[str, Any]        = 'left'
 def __init__(   self				, lowerCAmelCase__				, lowerCAmelCase__=False				, lowerCAmelCase__=True				, lowerCAmelCase__=False				, lowerCAmelCase__="<s>"				, lowerCAmelCase__="</s>"				, lowerCAmelCase__="<unk>"				, lowerCAmelCase__="<sep>"				, lowerCAmelCase__="<pad>"				, lowerCAmelCase__="<cls>"				, lowerCAmelCase__="<mask>"				, lowerCAmelCase__=["<eop>", "<eod>"]				, lowerCAmelCase__ = None				, **lowerCAmelCase__				, )					->				None:




        '''simple docstring'''
        # Mask token behave like a normal word, i.e. include the space before it
        lowercase__:   str  			=   AddedToken(lowerCAmelCase__				, lstrip=lowerCAmelCase__				, rstrip=lowerCAmelCase__ ) if isinstance(lowerCAmelCase__				, lowerCAmelCase__ ) else mask_token

        lowercase__:   List[Any]  			=   {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            do_lower_case=lowerCAmelCase__				, remove_space=lowerCAmelCase__				, keep_accents=lowerCAmelCase__				, bos_token=lowerCAmelCase__				, eos_token=lowerCAmelCase__				, unk_token=lowerCAmelCase__				, sep_token=lowerCAmelCase__				, pad_token=lowerCAmelCase__				, cls_token=lowerCAmelCase__				, mask_token=lowerCAmelCase__				, additional_special_tokens=lowerCAmelCase__				, sp_model_kwargs=self.sp_model_kwargs				, **lowerCAmelCase__				, )

        lowercase__:   Dict  			=   3

        lowercase__:   str  			=   do_lower_case
        lowercase__:   List[Any]  			=   remove_space
        lowercase__:   Union[str, Any]  			=   keep_accents
        lowercase__:   List[str]  			=   vocab_file

        lowercase__:   Union[str, Any]  			=   spm.SentencePieceProcessor(**self.sp_model_kwargs )
        self.sp_model.Load(lowerCAmelCase__ )
 @property
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Any:




        '''simple docstring'''
        return len(self.sp_model )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Any:




        '''simple docstring'''
        lowercase__:   Optional[Any]  			=   {self.convert_ids_to_tokens(lowerCAmelCase__ ): i for i in range(self.vocab_size )}
        vocab.update(self.added_tokens_encoder )
        return vocab
 def __getstate__(   self )					->				List[str]:




        '''simple docstring'''
        lowercase__:   Dict  			=   self.__dict__.copy()
        lowercase__:   Dict  			=   None
        return state
 def __setstate__(   self				, lowerCAmelCase__ )					->				Dict:




        '''simple docstring'''
        lowercase__:   Any  			=   d

        # for backward compatibility
        if not hasattr(self				, 'sp_model_kwargs' ):
               lowercase__:   Any  			=   {}

        lowercase__:   Dict  			=   spm.SentencePieceProcessor(**self.sp_model_kwargs )
        self.sp_model.Load(self.vocab_file )
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__ )					->				Tuple:




        '''simple docstring'''
        if self.remove_space:
               lowercase__:   Any  			=   ' '.join(inputs.strip().split() )
        else:
               lowercase__:   str  			=   inputs
        lowercase__:   List[str]  			=   outputs.replace('``'				, '"' ).replace('\'\''				, '"' )

        if not self.keep_accents:
               lowercase__:   List[str]  			=   unicodedata.normalize('NFKD'				, lowerCAmelCase__ )
               lowercase__:   Dict  			=   ''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__ )] )
        if self.do_lower_case:
               lowercase__:   Optional[Any]  			=   outputs.lower()

        return outputs
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__ )					->				List[str]:




        '''simple docstring'''
        lowercase__:   str  			=   self.preprocess_text(lowerCAmelCase__ )
        lowercase__:   Tuple  			=   self.sp_model.encode(lowerCAmelCase__				, out_type=lowerCAmelCase__ )
        lowercase__:   int  			=   []
        for piece in pieces:
               if len(lowerCAmelCase__ ) > 1 and piece[-1] == str(',' ) and piece[-2].isdigit():
                      lowercase__:   Tuple  			=   self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__				, '' ) )
                      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                             if len(cur_pieces[0] ) == 1:
                                    lowercase__:   List[str]  			=   cur_pieces[1:]
                             else:
                                    lowercase__:   Any  			=   cur_pieces[0][1:]
                      cur_pieces.append(piece[-1] )
                      new_pieces.extend(lowerCAmelCase__ )
               else:
                      new_pieces.append(lowerCAmelCase__ )

        return new_pieces
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__ )					->				Optional[Any]:




        '''simple docstring'''
        return self.sp_model.PieceToId(lowerCAmelCase__ )
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__ )					->				int:




        '''simple docstring'''
        return self.sp_model.IdToPiece(lowerCAmelCase__ )
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__ )					->				Any:




        '''simple docstring'''
        lowercase__:   List[Any]  			=   ''.join(lowerCAmelCase__ ).replace(lowerCAmelCase__				, ' ' ).strip()
        return out_string
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = False				, lowerCAmelCase__ = None				, lowerCAmelCase__ = True				, **lowerCAmelCase__				, )					->				str:




        '''simple docstring'''
        lowercase__:   List[str]  			=   kwargs.pop('use_source_tokenizer'				, lowerCAmelCase__ )

        lowercase__:   Any  			=   self.convert_ids_to_tokens(lowerCAmelCase__				, skip_special_tokens=lowerCAmelCase__ )

        # To avoid mixing byte-level and unicode for byte-level BPT
        # we need to build string separately for added tokens and byte-level tokens
        # cf. https://github.com/huggingface/transformers/issues/1133
        lowercase__:   Any  			=   []
        lowercase__:   List[Any]  			=   []
        for token in filtered_tokens:
               if skip_special_tokens and token in self.all_special_ids:
                      continue
               if token in self.added_tokens_encoder:
                      if current_sub_text:
                             sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__ ) )
                             lowercase__:   Union[str, Any]  			=   []
                      sub_texts.append(lowerCAmelCase__ )
               else:
                      current_sub_text.append(lowerCAmelCase__ )
        if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__ ) )

        # Mimic the behavior of the Rust tokenizer:
        # By default, there are no spaces between special tokens
        lowercase__:   Optional[Any]  			=   ''.join(lowerCAmelCase__ )

        lowercase__:   Optional[int]  			=   (
            clean_up_tokenization_spaces
            if clean_up_tokenization_spaces is not None
            else self.clean_up_tokenization_spaces
        )
        if clean_up_tokenization_spaces:
               lowercase__:   List[str]  			=   self.clean_up_tokenization(lowerCAmelCase__ )
               return clean_text
        else:
               return text
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				List[int]:




        '''simple docstring'''
        lowercase__:   List[Any]  			=   [self.sep_token_id]
        lowercase__:   Dict  			=   [self.cls_token_id]
        if token_ids_a is None:
               return token_ids_a + sep + cls
        return token_ids_a + sep + token_ids_a + sep + cls
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None				, lowerCAmelCase__ = False )					->				List[int]:




        '''simple docstring'''

        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCAmelCase__				, token_ids_a=lowerCAmelCase__				, already_has_special_tokens=lowerCAmelCase__ )

        if token_ids_a is not None:
               return ([0] * len(lowerCAmelCase__ )) + [1] + ([0] * len(lowerCAmelCase__ )) + [1, 1]
        return ([0] * len(lowerCAmelCase__ )) + [1, 1]
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				List[int]:




        '''simple docstring'''
        lowercase__:   Tuple  			=   [self.sep_token_id]
        lowercase__:   int  			=   [2]

        if token_ids_a is None:
               return len(token_ids_a + sep ) * [0] + cls_segment_id
        return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ = None )					->				Tuple[str]:




        '''simple docstring'''
        if not os.path.isdir(lowerCAmelCase__ ):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
               return
        lowercase__:   int  			=   os.path.join(
            lowerCAmelCase__				, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )

        if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase__ ) and os.path.isfile(self.vocab_file ):
               copyfile(self.vocab_file				, lowerCAmelCase__ )
        elif not os.path.isfile(self.vocab_file ):
               with open(lowerCAmelCase__				, 'wb' ) as fi:
                      lowercase__:   List[str]  			=   self.sp_model.serialized_model_proto()
                      fi.write(lowerCAmelCase__ )

        return (out_vocab_file,)






def    snake_case_     (  snake_case			,							snake_case     )  ->				str:
       if number < 0 or shift_amount < 0:
              raise ValueError('both inputs must be positive integers'     )

       lowercase__:   str  			=   str(bin(snake_case     )     )
       binary_number += "0" * shift_amount
       return binary_number




def    snake_case_     (  snake_case			,							snake_case     )  ->				str:
       if number < 0 or shift_amount < 0:
              raise ValueError('both inputs must be positive integers'     )

       lowercase__:   Optional[Any]  			=   str(bin(snake_case     )     )[2:]
       if shift_amount >= len(snake_case     ):
              return "0b0"
       lowercase__:   Optional[int]  			=   binary_number[: len(snake_case     ) - shift_amount]
       return "0b" + shifted_binary_number




def    snake_case_     (  snake_case			,							snake_case     )  ->				str:
       if number >= 0:  # Get binary representation of positive number
              lowercase__:   Union[str, Any]  			=   '0' + str(bin(snake_case     )     ).strip('-'     )[2:]
       else:  # Get binary (2's complement) representation of negative number
              lowercase__:   Dict  			=   len(bin(snake_case     )[3:]     )  # Find 2's complement of number
              lowercase__:   int  			=   bin(abs(snake_case     ) - (1 << binary_number_length)     )[3:]
              lowercase__:   Any  			=   (
                  '1' + '0' * (binary_number_length - len(snake_case     )) + binary_number
              )

       if shift_amount >= len(snake_case     ):
              return "0b" + binary_number[0] * len(snake_case     )
       return (
           "0b"
           + binary_number[0] * shift_amount
           + binary_number[: len(snake_case     ) - shift_amount]
       )


if __name__ == "__main__":
 import doctest

 doctest.testmod()


