



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__snake_case						:Optional[Any]      		=     logging.get_logger(__name__)

__snake_case						:Any      		=     {'''vocab_file''': '''spiece.model'''}

__snake_case						:Optional[Any]      		=     {
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}

class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     Optional[int]     ,		__SCREAMING_SNAKE_CASE							:     List[Any]     ,		__SCREAMING_SNAKE_CASE							:     int=False     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=True     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=False     ,		__SCREAMING_SNAKE_CASE							:     Dict="<s>"     ,		__SCREAMING_SNAKE_CASE							:     int="</s>"     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]="<unk>"     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]="<sep>"     ,		__SCREAMING_SNAKE_CASE							:     str="<pad>"     ,		__SCREAMING_SNAKE_CASE							:     Tuple="<cls>"     ,		__SCREAMING_SNAKE_CASE							:     int="<mask>"     ,		__SCREAMING_SNAKE_CASE							:     Dict=["<eop>", "<eod>"]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Dict[str, Any]] = None     ,		**__SCREAMING_SNAKE_CASE							:     str     ,		):


           '''simple docstring'''

           __a						       =				AddedToken(__SCREAMING_SNAKE_CASE     ,		lstrip=__SCREAMING_SNAKE_CASE     ,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE) else mask_token

           __a						       =				{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=__SCREAMING_SNAKE_CASE     ,		remove_space=__SCREAMING_SNAKE_CASE     ,		keep_accents=__SCREAMING_SNAKE_CASE     ,		bos_token=__SCREAMING_SNAKE_CASE     ,		eos_token=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		additional_special_tokens=__SCREAMING_SNAKE_CASE     ,		sp_model_kwargs=self.sp_model_kwargs     ,		**__SCREAMING_SNAKE_CASE     ,		)

           __a						       =				3

           __a						       =				do_lower_case
           __a						       =				remove_space
           __a						       =				keep_accents
           __a						       =				vocab_file

           __a						       =				spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(__SCREAMING_SNAKE_CASE)

           try:
               import jieba
           except ModuleNotFoundError as error:
               raise error.__class__(
                   '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
                   '''See https://pypi.org/project/jieba/ for installation.''')
           __a						       =				jieba
           __a						       =				str.maketrans(''' \n'''     ,		'''\u2582\u2583''')



       @property
       # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
       def 		_lowerCamelCase			(		self							:     Optional[int]):


           '''simple docstring'''

           return len(self.sp_model)



       def 		_lowerCamelCase			(		self							:     str):


           '''simple docstring'''

           __a						       =				{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
           vocab.update(self.added_tokens_encoder)
           return vocab



       def __getstate__(		self							:     str):


           '''simple docstring'''

           __a						       =				self.__dict__.copy()
           __a						       =				None
           return state



       def __setstate__(		self							:     Union[str, Any]     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]):


           '''simple docstring'''

           __a						       =				d

           # for backward compatibility
           if not hasattr(self     ,		'''sp_model_kwargs'''):
               __a						       =				{}

           __a						       =				spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(self.vocab_file)



       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[Any]):


           '''simple docstring'''

           if self.remove_space:
               __a						       =				''' '''.join(inputs.strip().split())
           else:
               __a						       =				inputs
           __a						       =				outputs.replace('''``'''     ,		'''"''').replace('''\'\''''     ,		'''"''')

           if not self.keep_accents:
               __a						       =				unicodedata.normalize('''NFKD'''     ,		__SCREAMING_SNAKE_CASE)
               __a						       =				''''''.join([c for c in outputs if not unicodedata.combining(__SCREAMING_SNAKE_CASE)])
           if self.do_lower_case:
               __a						       =				outputs.lower()

           return outputs



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     str):


           '''simple docstring'''

           __a						       =				self.preprocess_text(__SCREAMING_SNAKE_CASE)
           __a						       =				self.sp_model.encode(__SCREAMING_SNAKE_CASE     ,		out_type=__SCREAMING_SNAKE_CASE)
           __a						       =				[]
           for piece in pieces:
               if len(__SCREAMING_SNAKE_CASE) > 1 and piece[-1] == str(''',''') and piece[-2].isdigit():
                   __a						       =				self.sp_model.EncodeAsPieces(piece[:-1].replace(__SCREAMING_SNAKE_CASE     ,		''''''))
                   if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                       if len(cur_pieces[0]) == 1:
                           __a						       =				cur_pieces[1:]
                       else:
                           __a						       =				cur_pieces[0][1:]
                   cur_pieces.append(piece[-1])
                   new_pieces.extend(__SCREAMING_SNAKE_CASE)
               else:
                   new_pieces.append(__SCREAMING_SNAKE_CASE)

           return new_pieces



       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]):


           '''simple docstring'''

           return self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Union[str, Any]     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]):


           '''simple docstring'''

           return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[str]):


           '''simple docstring'''

           __a						       =				''''''.join(__SCREAMING_SNAKE_CASE).replace(__SCREAMING_SNAKE_CASE     ,		''' ''').strip()
           return out_string



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[self.cls_token_id]
           if token_ids_a is None:
               return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None     ,		__SCREAMING_SNAKE_CASE							:     bool = False):


           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__SCREAMING_SNAKE_CASE     ,		token_ids_a=__SCREAMING_SNAKE_CASE     ,		already_has_special_tokens=__SCREAMING_SNAKE_CASE)

           if token_ids_a is not None:
               return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1]
           return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1]



       def 		_lowerCamelCase			(		self							:     Optional[int]     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[2]

           if token_ids_a is None:
               return len(token_ids_a + sep) * [0] + cls_segment_id
           return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id



       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           if not os.path.isdir(__SCREAMING_SNAKE_CASE):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
           __a						       =				os.path.join(
               __SCREAMING_SNAKE_CASE     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE) and os.path.isfile(self.vocab_file):
               copyfile(self.vocab_file     ,		__SCREAMING_SNAKE_CASE)
           elif not os.path.isfile(self.vocab_file):
               with open(__SCREAMING_SNAKE_CASE     ,		'''wb''') as fi:
                   __a						       =				self.sp_model.serialized_model_proto()
                   fi.write(__SCREAMING_SNAKE_CASE)

           return (out_vocab_file,)





       def 		_lowerCamelCase			(		self							:     Any     ,		*__SCREAMING_SNAKE_CASE							:     int     ,		**__SCREAMING_SNAKE_CASE							:     List[Any]):


           '''simple docstring'''

           __a						       =				super()._decode(*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
           __a						       =				text.replace(''' '''     ,		'''''').replace('''\u2582'''     ,		''' ''').replace('''\u2583'''     ,		'''\n''')
           return text






'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A			:      int					      =				logging.get_logger(__name__)

_A			:      Any					      =				{'''ctrl''': '''https://huggingface.co/ctrl/resolve/main/config.json'''}







class 				_lowercase     (				UpperCAmelCase__			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       Any      	=					"""ctrl"""
 _SCREAMING_SNAKE_CASE     :       Optional[Any]      	=					["""past_key_values"""]
 _SCREAMING_SNAKE_CASE     :       Union[str, Any]      	=					{
     """max_position_embeddings""": """n_positions""",
     """hidden_size""": """n_embd""",
     """num_attention_heads""": """n_head""",
     """num_hidden_layers""": """n_layer""",
 }
 def __init__(      self   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]=24_65_34      ,       SCREAMING_SNAKE_CASE__   :	Optional[int]=2_56      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]=12_80      ,       SCREAMING_SNAKE_CASE__   :	str=81_92      ,       SCREAMING_SNAKE_CASE__   :	List[str]=48      ,       SCREAMING_SNAKE_CASE__   :	Tuple=16      ,       SCREAMING_SNAKE_CASE__   :	str=0.1      ,       SCREAMING_SNAKE_CASE__   :	Dict=0.1      ,       SCREAMING_SNAKE_CASE__   :	Dict=1e-6      ,       SCREAMING_SNAKE_CASE__   :	Any=0.0_2      ,       SCREAMING_SNAKE_CASE__   :	Tuple=True      ,       **SCREAMING_SNAKE_CASE__   :	Optional[int]      ,       )  ->    Tuple:
  __lowerCAmelCase											=						vocab_size
  __lowerCAmelCase											=						n_positions
  __lowerCAmelCase											=						n_embd
  __lowerCAmelCase											=						n_layer
  __lowerCAmelCase											=						n_head
  __lowerCAmelCase											=						dff
  __lowerCAmelCase											=						resid_pdrop
  __lowerCAmelCase											=						embd_pdrop
  __lowerCAmelCase											=						layer_norm_epsilon
  __lowerCAmelCase											=						initializer_range

  __lowerCAmelCase											=						use_cache

  super().__init__(**SCREAMING_SNAKE_CASE__							)






