



"""simple docstring"""
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__a      =				logging.get_logger(__name__)

__a      =				{"vocab_file": "spiece.model"}

__a      =				{
    "vocab_file": {
        "TsinghuaAI/CPM-Generate": "https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model",
    }
}
class 				lowerCamelCase  (			_lowerCAmelCase				):






					'''simple docstring'''





					def __init__(       self:							Tuple						,     snake_case:							Any						,     snake_case:							Any=False						,     snake_case:							Any=True						,     snake_case:							List[Any]=False						,     snake_case:							Tuple="<s>"						,     snake_case:							List[str]="</s>"						,     snake_case:							Tuple="<unk>"						,     snake_case:							Optional[Any]="<sep>"						,     snake_case:							Any="<pad>"						,     snake_case:							Optional[Any]="<cls>"						,     snake_case:							int="<mask>"						,     snake_case:							List[Any]=["<eop>", "<eod>"]						,     snake_case:							Optional[Dict[str, Any]] = None						,     **snake_case:							int						,     )   ->	None:
											snake_case_ :Dict						=			AddedToken(snake_case						,     lstrip=snake_case						,     rstrip=snake_case						) if isinstance(snake_case						,     snake_case						) else mask_token

											snake_case_ :int						=			{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=snake_case						,     remove_space=snake_case						,     keep_accents=snake_case						,     bos_token=snake_case						,     eos_token=snake_case						,     unk_token=snake_case						,     sep_token=snake_case						,     pad_token=snake_case						,     cls_token=snake_case						,     mask_token=snake_case						,     additional_special_tokens=snake_case						,     sp_model_kwargs=self.sp_model_kwargs						,     **snake_case						,     )

											snake_case_ :str						=			3

											snake_case_ :List[Any]						=			do_lower_case
											snake_case_ :List[str]						=			remove_space
											snake_case_ :str						=			keep_accents
											snake_case_ :str						=			vocab_file

											snake_case_ :Any						=			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(snake_case						)

											try:
																	import jieba
											except ModuleNotFoundError as error:
																	raise error.__class__(
																	    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
																	    """See https://pypi.org/project/jieba/ for installation."""						)
											snake_case_ :List[str]						=			jieba
											snake_case_ :str						=			str.maketrans(""" \n"""						,     """\u2582\u2583"""						)






					@property
					# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
					def    lowerCAmelCase_				(       self:							Dict						)   ->	Tuple:
											return len(self.sp_model						)






					def    lowerCAmelCase_				(       self:							int						)   ->	Any:
											snake_case_ :Any						=			{self.convert_ids_to_tokens(snake_case						): i for i in range(self.vocab_size						)}
											vocab.update(self.added_tokens_encoder						)
											return vocab






					def __getstate__(       self:							Dict						)   ->	Dict:
											snake_case_ :Optional[Any]						=			self.__dict__.copy()
											snake_case_ :Dict						=			None
											return state






					def __setstate__(       self:							int						,     snake_case:							str						)   ->	Tuple:
											snake_case_ :Tuple						=			d

											# for backward compatibility
											if not hasattr(self						,     """sp_model_kwargs"""						):
																	snake_case_ :int						=			{}

											snake_case_ :Optional[Any]						=			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(self.vocab_file						)






					def    lowerCAmelCase_				(       self:							Any						,     snake_case:							List[Any]						)   ->	str:
											if self.remove_space:
																	snake_case_ :Dict						=			""" """.join(inputs.strip().split()						)
											else:
																	snake_case_ :Any						=			inputs
											snake_case_ :Optional[Any]						=			outputs.replace("""``"""						,     """\""""						).replace("""''"""						,     """\""""						)

											if not self.keep_accents:
																	snake_case_ :Any						=			unicodedata.normalize("""NFKD"""						,     snake_case						)
																	snake_case_ :int						=			"""""".join([c for c in outputs if not unicodedata.combining(snake_case						)]						)
											if self.do_lower_case:
																	snake_case_ :Tuple						=			outputs.lower()

											return outputs






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							str						)   ->	List[str]:
											snake_case_ :str						=			self.preprocess_text(snake_case						)
											snake_case_ :Optional[Any]						=			self.sp_model.encode(snake_case						,     out_type=snake_case						)
											snake_case_ :int						=			[]
											for piece in pieces:
																	if len(snake_case						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
																							snake_case_ :List[str]						=			self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case						,     """"""						)						)
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																													if len(cur_pieces[0]						) == 1:
																																			snake_case_ :Any						=			cur_pieces[1:]
																													else:
																																			snake_case_ :Optional[int]						=			cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]						)
																							new_pieces.extend(snake_case						)
																	else:
																							new_pieces.append(snake_case						)

											return new_pieces






					def    lowerCAmelCase_				(       self:							int						,     snake_case:							str						)   ->	Dict:
											return self.sp_model.PieceToId(snake_case						)






					def    lowerCAmelCase_				(       self:							Dict						,     snake_case:							int						)   ->	List[str]:
											return self.sp_model.IdToPiece(snake_case						)






					def    lowerCAmelCase_				(       self:							Union[str, Any]						,     snake_case:							Union[str, Any]						)   ->	List[Any]:
											snake_case_ :str						=			"""""".join(snake_case						).replace(snake_case						,     """ """						).strip()
											return out_string






					def    lowerCAmelCase_				(       self:							int						,     snake_case:							List[int]						,     snake_case:							Optional[List[int]] = None						)   ->	List[int]:
											snake_case_ :Tuple						=			[self.sep_token_id]
											snake_case_ :Tuple						=			[self.cls_token_id]
											if token_ids_a is None:
																	return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls






					def    lowerCAmelCase_				(       self:							int						,     snake_case:							List[int]						,     snake_case:							Optional[List[int]] = None						,     snake_case:							bool = False						)   ->	List[int]:

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=snake_case						,     token_ids_a=snake_case						,     already_has_special_tokens=snake_case						)

											if token_ids_a is not None:
																	return ([0] * len(snake_case						)) + [1] + ([0] * len(snake_case						)) + [1, 1]
											return ([0] * len(snake_case						)) + [1, 1]






					def    lowerCAmelCase_				(       self:							int						,     snake_case:							List[int]						,     snake_case:							Optional[List[int]] = None						)   ->	List[int]:
											snake_case_ :List[Any]						=			[self.sep_token_id]
											snake_case_ :str						=			[2]

											if token_ids_a is None:
																	return len(token_ids_a + sep						) * [0] + cls_segment_id
											return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id






					def    lowerCAmelCase_				(       self:							Union[str, Any]						,     snake_case:							str						,     snake_case:							Optional[str] = None						)   ->	Tuple[str]:
											if not os.path.isdir(snake_case						):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
																	return
											snake_case_ :List[Any]						=			os.path.join(
											    snake_case						,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

											if os.path.abspath(self.vocab_file						) != os.path.abspath(snake_case						) and os.path.isfile(self.vocab_file						):
																	copyfile(self.vocab_file						,     snake_case						)
											elif not os.path.isfile(self.vocab_file						):
																	with open(snake_case						,     """wb"""						) as fi:
																							snake_case_ :Union[str, Any]						=			self.sp_model.serialized_model_proto()
																							fi.write(snake_case						)

											return (out_vocab_file,)






					def    lowerCAmelCase_				(       self:							List[str]						,     *snake_case:							str						,     **snake_case:							List[Any]						)   ->	Tuple:
											snake_case_ :int						=			super()._decode(*snake_case						,     **snake_case						)
											snake_case_ :Optional[int]						=			text.replace(""" """						,     """"""						).replace("""\u2582"""						,     """ """						).replace("""\u2583"""						,     """\n"""						)
											return text








class  UpperCamelCase_      :



	'''simple docstring'''




	def __init__(       self       ,			a					)       ->    None:
								snake_case_				=							set_counts
								snake_case_				=							max(a					)
								snake_case_				=							len(a					)
								snake_case_				=							[1] * num_sets
								snake_case_				=							list(range(a					)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a					)       ->    bool:
								snake_case_				=							self.get_parent(a					)
								snake_case_				=							self.get_parent(a					)

								if src_parent == dst_parent:
															return False

								if self.ranks[dst_parent] >= self.ranks[src_parent]:
															self.set_counts[dst_parent] += self.set_counts[src_parent]
															snake_case_				=							0
															snake_case_				=							dst_parent
															if self.ranks[dst_parent] == self.ranks[src_parent]:
																						self.ranks[dst_parent] += 1
															snake_case_				=							self.set_counts[dst_parent]
								else:
															self.set_counts[src_parent] += self.set_counts[dst_parent]
															snake_case_				=							0
															snake_case_				=							src_parent
															snake_case_				=							self.set_counts[src_parent]

								snake_case_				=							max(self.max_set       ,			a					)
								return True






	def 							_UpperCamelCase			(       self       ,			a					)       ->    int:
								if self.parents[disj_set] == disj_set:
															return disj_set
								snake_case_				=							self.get_parent(self.parents[disj_set]					)
								return self.parents[disj_set]





