



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A__	      =       logging.get_logger(__name__)

A__	      =       {"""vocab_file""": """spiece.model"""}

A__	      =       {
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    }
}





class 					__lowerCAmelCase (     lowerCamelCase__       ):




				def __init__(							self						,				_snake_case						,				_snake_case=False						,				_snake_case=True						,				_snake_case=False						,				_snake_case="<s>"						,				_snake_case="</s>"						,				_snake_case="<unk>"						,				_snake_case="<sep>"						,				_snake_case="<pad>"						,				_snake_case="<cls>"						,				_snake_case="<mask>"						,				_snake_case=["<eop>", "<eod>"]						,				_snake_case = None						,				**_snake_case						,				):



									"""simple docstring"""





									_lowerCAmelCase						=    AddedToken(_snake_case						,				lstrip=_snake_case						,				rstrip=_snake_case		) if isinstance(_snake_case						,				_snake_case		) else mask_token

									_lowerCAmelCase						=    {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_snake_case						,				remove_space=_snake_case						,				keep_accents=_snake_case						,				bos_token=_snake_case						,				eos_token=_snake_case						,				unk_token=_snake_case						,				sep_token=_snake_case						,				pad_token=_snake_case						,				cls_token=_snake_case						,				mask_token=_snake_case						,				additional_special_tokens=_snake_case						,				sp_model_kwargs=self.sp_model_kwargs						,				**_snake_case						,				)

									_lowerCAmelCase						=    3

									_lowerCAmelCase						=    do_lower_case
									_lowerCAmelCase						=    remove_space
									_lowerCAmelCase						=    keep_accents
									_lowerCAmelCase						=    vocab_file

									_lowerCAmelCase						=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
									self.sp_model.Load(_snake_case		)

									try:
														import jieba
									except ModuleNotFoundError as error:
														raise error.__class__(
														    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
														    """See https://pypi.org/project/jieba/ for installation."""		)
									_lowerCAmelCase						=    jieba
									_lowerCAmelCase						=    str.maketrans(""" \n"""						,				"""\u2582\u2583"""		)




				@property
				# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
				def 							snake_case   (							self		):



									"""simple docstring"""





									return len(self.sp_model		)




				def 							snake_case   (							self		):



									"""simple docstring"""





									_lowerCAmelCase						=    {self.convert_ids_to_tokens(_snake_case		): i for i in range(self.vocab_size		)}
									vocab.update(self.added_tokens_encoder		)
									return vocab




				def __getstate__(							self		):



									"""simple docstring"""





									_lowerCAmelCase						=    self.__dict__.copy()
									_lowerCAmelCase						=    None
									return state




				def __setstate__(							self						,				_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase						=    d

									# for backward compatibility
									if not hasattr(self						,				"""sp_model_kwargs"""		):
														_lowerCAmelCase						=    {}

									_lowerCAmelCase						=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
									self.sp_model.Load(self.vocab_file		)




				def 							snake_case   (							self						,				_snake_case		):



									"""simple docstring"""





									if self.remove_space:
														_lowerCAmelCase						=    """ """.join(inputs.strip().split()		)
									else:
														_lowerCAmelCase						=    inputs
									_lowerCAmelCase						=    outputs.replace("""``"""						,				"""\""""		).replace("""''"""						,				"""\""""		)

									if not self.keep_accents:
														_lowerCAmelCase						=    unicodedata.normalize("""NFKD"""						,				_snake_case		)
														_lowerCAmelCase						=    """""".join([c for c in outputs if not unicodedata.combining(_snake_case		)]		)
									if self.do_lower_case:
														_lowerCAmelCase						=    outputs.lower()

									return outputs




				def 							snake_case   (							self						,				_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase						=    self.preprocess_text(_snake_case		)
									_lowerCAmelCase						=    self.sp_model.encode(_snake_case						,				out_type=_snake_case		)
									_lowerCAmelCase						=    []
									for piece in pieces:
														if len(_snake_case		) > 1 and piece[-1] == str(""","""		) and piece[-2].isdigit():
																			_lowerCAmelCase						=    self.sp_model.EncodeAsPieces(piece[:-1].replace(_snake_case						,				""""""		)		)
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																								if len(cur_pieces[0]		) == 1:
																													_lowerCAmelCase						=    cur_pieces[1:]
																								else:
																													_lowerCAmelCase						=    cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]		)
																			new_pieces.extend(_snake_case		)
														else:
																			new_pieces.append(_snake_case		)

									return new_pieces




				def 							snake_case   (							self						,				_snake_case		):



									"""simple docstring"""





									return self.sp_model.PieceToId(_snake_case		)




				def 							snake_case   (							self						,				_snake_case		):



									"""simple docstring"""





									return self.sp_model.IdToPiece(_snake_case		)




				def 							snake_case   (							self						,				_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase						=    """""".join(_snake_case		).replace(_snake_case						,				""" """		).strip()
									return out_string




				def 							snake_case   (							self						,				_snake_case						,				_snake_case = None		):



									"""simple docstring"""





									_lowerCAmelCase						=    [self.sep_token_id]
									_lowerCAmelCase						=    [self.cls_token_id]
									if token_ids_a is None:
														return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls




				def 							snake_case   (							self						,				_snake_case						,				_snake_case = None						,				_snake_case = False		):



									"""simple docstring"""





									if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_snake_case						,				token_ids_a=_snake_case						,				already_has_special_tokens=_snake_case		)

									if token_ids_a is not None:
														return ([0] * len(_snake_case		)) + [1] + ([0] * len(_snake_case		)) + [1, 1]
									return ([0] * len(_snake_case		)) + [1, 1]




				def 							snake_case   (							self						,				_snake_case						,				_snake_case = None		):



									"""simple docstring"""





									_lowerCAmelCase						=    [self.sep_token_id]
									_lowerCAmelCase						=    [2]

									if token_ids_a is None:
														return len(token_ids_a + sep		) * [0] + cls_segment_id
									return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id




				def 							snake_case   (							self						,				_snake_case						,				_snake_case = None		):



									"""simple docstring"""





									if not os.path.isdir(_snake_case		):
														logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
														return
									_lowerCAmelCase						=    os.path.join(
									    _snake_case						,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

									if os.path.abspath(self.vocab_file		) != os.path.abspath(_snake_case		) and os.path.isfile(self.vocab_file		):
														copyfile(self.vocab_file						,				_snake_case		)
									elif not os.path.isfile(self.vocab_file		):
														with open(_snake_case						,				"""wb"""		) as fi:
																			_lowerCAmelCase						=    self.sp_model.serialized_model_proto()
																			fi.write(_snake_case		)

									return (out_vocab_file,)




				def 							snake_case   (							self						,				*_snake_case						,				**_snake_case		):



									"""simple docstring"""





									_lowerCAmelCase						=    super()._decode(*_snake_case						,				**_snake_case		)
									_lowerCAmelCase						=    text.replace(""" """						,				""""""		).replace("""\u2582"""						,				""" """		).replace("""\u2583"""						,				"""\n"""		)
									return text







from abc import ABC, abstractmethod
from argparse import ArgumentParser







class        snake_case_   (lowerCamelCase_      ):
					@staticmethod
					@abstractmethod
					def  lowerCamelCase__(      __snake_case   :ArgumentParser      )					->    Dict:
												raise NotImplementedError()



					@abstractmethod
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    Dict:
												raise NotImplementedError()




