



'''simple docstring'''


from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_lowerCamelCase	:       str      	=	logging.get_logger(__name__)

_lowerCamelCase	:       Union[str, Any]      	=	{
    "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/config.json",
    # See all Marian models at https://huggingface.co/models?filter=marian
}

class  SCREAMING_SNAKE_CASE (  _a		):






    """simple docstring"""





    _SCREAMING_SNAKE_CASE	  =  """marian"""
    _SCREAMING_SNAKE_CASE	  =  ["""past_key_values"""]
    _SCREAMING_SNAKE_CASE	  =  {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}






    def __init__(  self      :					List[str]  ,						UpperCamelCase__      :					Optional[Any]=5_8_1_0_1  ,						UpperCamelCase__      :					Dict=None  ,						UpperCamelCase__      :					Any=1_0_2_4  ,						UpperCamelCase__      :					List[str]=1_2  ,						UpperCamelCase__      :					Dict=4_0_9_6  ,						UpperCamelCase__      :					Tuple=1_6  ,						UpperCamelCase__      :					Any=1_2  ,						UpperCamelCase__      :					List[str]=4_0_9_6  ,						UpperCamelCase__      :					int=1_6  ,						UpperCamelCase__      :					str=0.0  ,						UpperCamelCase__      :					Optional[Any]=0.0  ,						UpperCamelCase__      :					Optional[int]=True  ,						UpperCamelCase__      :					List[Any]=True  ,						UpperCamelCase__      :					Optional[int]="gelu"  ,						UpperCamelCase__      :					List[Any]=1_0_2_4  ,						UpperCamelCase__      :					str=0.1  ,						UpperCamelCase__      :					Dict=0.0  ,						UpperCamelCase__      :					Tuple=0.0  ,						UpperCamelCase__      :					List[str]=0.0_2  ,						UpperCamelCase__      :					List[str]=5_8_1_0_0  ,						UpperCamelCase__      :					Dict=False  ,						UpperCamelCase__      :					Optional[int]=5_8_1_0_0  ,						UpperCamelCase__      :					Union[str, Any]=0  ,						UpperCamelCase__      :					List[str]=0  ,						UpperCamelCase__      :					int=True  ,						**UpperCamelCase__      :					Optional[Any]  ,						):


       """simple docstring"""







       UpperCamelCase					=  vocab_size
       UpperCamelCase					=  decoder_vocab_size or vocab_size
       UpperCamelCase					=  max_position_embeddings
       UpperCamelCase					=  d_model
       UpperCamelCase					=  encoder_ffn_dim
       UpperCamelCase					=  encoder_layers
       UpperCamelCase					=  encoder_attention_heads
       UpperCamelCase					=  decoder_ffn_dim
       UpperCamelCase					=  decoder_layers
       UpperCamelCase					=  decoder_attention_heads
       UpperCamelCase					=  dropout
       UpperCamelCase					=  attention_dropout
       UpperCamelCase					=  activation_dropout
       UpperCamelCase					=  activation_function
       UpperCamelCase					=  init_std
       UpperCamelCase					=  encoder_layerdrop
       UpperCamelCase					=  decoder_layerdrop
       UpperCamelCase					=  use_cache
       UpperCamelCase					=  encoder_layers
       UpperCamelCase					=  scale_embedding  # scale factor will be sqrt(d_model) if True
       UpperCamelCase					=  share_encoder_decoder_embeddings
       super().__init__(
           pad_token_id=UpperCamelCase__  ,						eos_token_id=UpperCamelCase__  ,						is_encoder_decoder=UpperCamelCase__  ,						decoder_start_token_id=UpperCamelCase__  ,						forced_eos_token_id=UpperCamelCase__  ,						**UpperCamelCase__  ,						)

class  SCREAMING_SNAKE_CASE (  _a		):






    """simple docstring"""





    @property
    # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.inputs
    def  A       (  self      :					List[str] ):


       """simple docstring"""







       if self.task in ["default", "seq2seq-lm"]:
          UpperCamelCase					=  OrderedDict(
              [
                  ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                  ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
              ] )

          if self.use_past:
             UpperCamelCase					=  {0: 'batch'}
             UpperCamelCase					=  {0: 'batch', 1: 'past_decoder_sequence + sequence'}
          else:
             UpperCamelCase					=  {0: 'batch', 1: 'decoder_sequence'}
             UpperCamelCase					=  {0: 'batch', 1: 'decoder_sequence'}

          if self.use_past:
             self.fill_with_past_key_values_(UpperCamelCase__  ,						direction='inputs' )
       elif self.task == "causal-lm":
          # TODO: figure this case out.
          UpperCamelCase					=  OrderedDict(
              [
                  ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                  ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
              ] )
          if self.use_past:
             UpperCamelCase				,      UpperCamelCase					=  self.num_layers
             for i in range(UpperCamelCase__ ):
                UpperCamelCase					=  {0: 'batch', 2: 'past_sequence + sequence'}
                UpperCamelCase					=  {0: 'batch', 2: 'past_sequence + sequence'}
       else:
          UpperCamelCase					=  OrderedDict(
              [
                  ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
                  ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
                  ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
                  ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
              ] )

       return common_inputs






    @property
    # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
    def  A       (  self      :					Dict ):


       """simple docstring"""







       if self.task in ["default", "seq2seq-lm"]:
          UpperCamelCase					=  super().outputs
       else:
          UpperCamelCase					=  super(UpperCamelCase__  ,						self ).outputs
          if self.use_past:
             UpperCamelCase				,      UpperCamelCase					=  self.num_layers
             for i in range(UpperCamelCase__ ):
                UpperCamelCase					=  {0: 'batch', 2: 'past_sequence + sequence'}
                UpperCamelCase					=  {0: 'batch', 2: 'past_sequence + sequence'}
       return common_outputs






    def  A       (  self      :					List[str]  ,						UpperCamelCase__      :					PreTrainedTokenizer  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					bool = False  ,						UpperCamelCase__      :					Optional[TensorType] = None  ,						):


       """simple docstring"""







       UpperCamelCase					=  self._generate_dummy_inputs_for_encoder_and_decoder(
           UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )

       # Generate decoder inputs
       UpperCamelCase					=  seq_length if not self.use_past else 1
       UpperCamelCase					=  self._generate_dummy_inputs_for_encoder_and_decoder(
           UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )
       UpperCamelCase					=  {f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
       UpperCamelCase					=  dict(**UpperCamelCase__  ,						**UpperCamelCase__ )

       if self.use_past:
          if not is_torch_available():
             raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.' )
          else:
             import torch
          UpperCamelCase				,      UpperCamelCase					=  common_inputs['input_ids'].shape
          UpperCamelCase					=  common_inputs['decoder_input_ids'].shape[1]
          UpperCamelCase				,      UpperCamelCase					=  self.num_attention_heads
          UpperCamelCase					=  (
              batch,
              num_encoder_attention_heads,
              encoder_seq_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )
          UpperCamelCase					=  decoder_seq_length + 3
          UpperCamelCase					=  (
              batch,
              num_decoder_attention_heads,
              decoder_past_length,
              self._config.hidden_size // num_decoder_attention_heads,
          )

          UpperCamelCase					=  torch.cat(
              [common_inputs['decoder_attention_mask'], torch.ones(UpperCamelCase__  ,						UpperCamelCase__ )]  ,						dim=1 )

          UpperCamelCase					=  []
          # If the number of encoder and decoder layers are present in the model configuration, both are considered
          UpperCamelCase				,      UpperCamelCase					=  self.num_layers
          UpperCamelCase					=  min(UpperCamelCase__  ,						UpperCamelCase__ )
          UpperCamelCase					=  max(UpperCamelCase__  ,						UpperCamelCase__ ) - min_num_layers
          UpperCamelCase					=  'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

          for _ in range(UpperCamelCase__ ):
             common_inputs["past_key_values"].append(
                 (
                     torch.zeros(UpperCamelCase__ ),
                     torch.zeros(UpperCamelCase__ ),
                     torch.zeros(UpperCamelCase__ ),
                     torch.zeros(UpperCamelCase__ ),
                 ) )
          # TODO: test this.
          UpperCamelCase					=  encoder_shape if remaining_side_name == 'encoder' else decoder_shape
          for _ in range(UpperCamelCase__  ,						UpperCamelCase__ ):
             common_inputs["past_key_values"].append((torch.zeros(UpperCamelCase__ ), torch.zeros(UpperCamelCase__ )) )
       return common_inputs






    def  A       (  self      :					Tuple  ,						UpperCamelCase__      :					PreTrainedTokenizer  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					bool = False  ,						UpperCamelCase__      :					Optional[TensorType] = None  ,						):


       """simple docstring"""







       UpperCamelCase					=  self._generate_dummy_inputs_for_encoder_and_decoder(
           UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )

       if self.use_past:
          if not is_torch_available():
             raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.' )
          else:
             import torch
          UpperCamelCase				,      UpperCamelCase					=  common_inputs['input_ids'].shape
          # Not using the same length for past_key_values
          UpperCamelCase					=  seqlen + 2
          UpperCamelCase				,      UpperCamelCase					=  self.num_layers
          UpperCamelCase				,      UpperCamelCase					=  self.num_attention_heads
          UpperCamelCase					=  (
              batch,
              num_encoder_attention_heads,
              past_key_values_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )

          UpperCamelCase					=  common_inputs['attention_mask'].dtype
          UpperCamelCase					=  torch.cat(
              [common_inputs['attention_mask'], torch.ones(UpperCamelCase__  ,						UpperCamelCase__  ,						dtype=UpperCamelCase__ )]  ,						dim=1 )
          UpperCamelCase					=  [
              (torch.zeros(UpperCamelCase__ ), torch.zeros(UpperCamelCase__ )) for _ in range(UpperCamelCase__ )
          ]
       return common_inputs






    def  A       (  self      :					int  ,						UpperCamelCase__      :					PreTrainedTokenizer  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					bool = False  ,						UpperCamelCase__      :					Optional[TensorType] = None  ,						):


       """simple docstring"""







       UpperCamelCase					=  compute_effective_axis_dimension(
           UpperCamelCase__  ,						fixed_dimension=OnnxConfig.default_fixed_batch  ,						num_token_to_add=0 )

       # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
       UpperCamelCase					=  tokenizer.num_special_tokens_to_add(UpperCamelCase__ )
       UpperCamelCase					=  compute_effective_axis_dimension(
           UpperCamelCase__  ,						fixed_dimension=OnnxConfig.default_fixed_sequence  ,						num_token_to_add=UpperCamelCase__ )

       # Generate dummy inputs according to compute batch and sequence
       UpperCamelCase					=  [' '.join([tokenizer.unk_token] ) * seq_length] * batch_size
       UpperCamelCase					=  dict(tokenizer(UpperCamelCase__  ,						return_tensors=UpperCamelCase__ ) )
       return common_inputs






    def  A       (  self      :					Dict  ,						UpperCamelCase__      :					PreTrainedTokenizer  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					int = -1  ,						UpperCamelCase__      :					bool = False  ,						UpperCamelCase__      :					Optional[TensorType] = None  ,						):


       """simple docstring"""







       if self.task in ["default", "seq2seq-lm"]:
          UpperCamelCase					=  self._generate_dummy_inputs_for_default_and_seqaseq_lm(
              UpperCamelCase__  ,						batch_size=UpperCamelCase__  ,						seq_length=UpperCamelCase__  ,						is_pair=UpperCamelCase__  ,						framework=UpperCamelCase__ )

       else:
          UpperCamelCase					=  self._generate_dummy_inputs_for_causal_lm(
              UpperCamelCase__  ,						batch_size=UpperCamelCase__  ,						seq_length=UpperCamelCase__  ,						is_pair=UpperCamelCase__  ,						framework=UpperCamelCase__ )

       return common_inputs






    def  A       (  self      :					List[str]  ,						UpperCamelCase__      :					Any  ,						UpperCamelCase__      :					List[str]  ,						UpperCamelCase__      :					Tuple  ,						UpperCamelCase__      :					Optional[int] ):


       """simple docstring"""







       if self.task in ["default", "seq2seq-lm"]:
          UpperCamelCase					=  super()._flatten_past_key_values_(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )
       else:
          UpperCamelCase					=  super(UpperCamelCase__  ,						self )._flatten_past_key_values_(
              UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )






    @property
    def  A       (  self      :					Dict ):


       """simple docstring"""







       return 1E-4






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A__								=						logging.get_logger(__name__)

A__								=						{'''vocab_file''': '''spiece.model'''}

A__								=						{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}


class       a							(			__lowerCamelCase				):



		def __init__(     self     :Union[str, Any]   ,__lowercase     :Optional[Any]   ,__lowercase     :int=False   ,__lowercase     :int=True   ,__lowercase     :Optional[int]=False   ,__lowercase     :List[str]="<s>"   ,__lowercase     :str="</s>"   ,__lowercase     :Dict="<unk>"   ,__lowercase     :Optional[int]="<sep>"   ,__lowercase     :Tuple="<pad>"   ,__lowercase     :Union[str, Any]="<cls>"   ,__lowercase     :Dict="<mask>"   ,__lowercase     :List[Any]=["<eop>", "<eod>"]   ,__lowercase     :Optional[Dict[str, Any]] = None   ,**__lowercase     :int   ,):
						snake_case__	:      Any     		=	AddedToken(__lowercase   ,lstrip=__lowercase   ,rstrip=__lowercase  ) if isinstance(__lowercase   ,__lowercase  ) else mask_token

						snake_case__	:      Tuple     		=	{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=__lowercase   ,remove_space=__lowercase   ,keep_accents=__lowercase   ,bos_token=__lowercase   ,eos_token=__lowercase   ,unk_token=__lowercase   ,sep_token=__lowercase   ,pad_token=__lowercase   ,cls_token=__lowercase   ,mask_token=__lowercase   ,additional_special_tokens=__lowercase   ,sp_model_kwargs=self.sp_model_kwargs   ,**__lowercase   ,)

						snake_case__	:      Optional[Any]     		=	3

						snake_case__	:      List[str]     		=	do_lower_case
						snake_case__	:      Union[str, Any]     		=	remove_space
						snake_case__	:      Tuple     		=	keep_accents
						snake_case__	:      List[str]     		=	vocab_file

						snake_case__	:      Optional[Any]     		=	spm.SentencePieceProcessor(**self.sp_model_kwargs  )
						self.sp_model.Load(__lowercase  )

						try:
										import jieba
						except ModuleNotFoundError as error:
										raise error.__class__(
										    '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
										    '''See https://pypi.org/project/jieba/ for installation.'''  )
						snake_case__	:      List[Any]     		=	jieba
						snake_case__	:      Union[str, Any]     		=	str.maketrans(''' \n'''   ,'''\u2582\u2583'''  )



		@property
		# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						return len(self.sp_model  )



		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Optional[Any]     		=	{self.convert_ids_to_tokens(__lowercase  ): i for i in range(self.vocab_size  )}
						vocab.update(self.added_tokens_encoder  )
						return vocab



		def __getstate__(     self     :List[str]  ):
						snake_case__	:      Optional[Any]     		=	self.__dict__.copy()
						snake_case__	:      Optional[int]     		=	None
						return state



		def __setstate__(     self     :int   ,__lowercase     :str  ):
						snake_case__	:      Tuple     		=	d

						# for backward compatibility
						if not hasattr(self   ,'''sp_model_kwargs'''  ):
										snake_case__	:      List[Any]     		=	{}

						snake_case__	:      List[Any]     		=	spm.SentencePieceProcessor(**self.sp_model_kwargs  )
						self.sp_model.Load(self.vocab_file  )



		def 	__lowerCamelCase			(     self     :Dict   ,__lowercase     :Optional[int]  ):
						if self.remove_space:
										snake_case__	:      int     		=	''' '''.join(inputs.strip().split()  )
						else:
										snake_case__	:      Tuple     		=	inputs
						snake_case__	:      List[Any]     		=	outputs.replace('''``'''   ,'''"'''  ).replace('''\'\''''   ,'''"'''  )

						if not self.keep_accents:
										snake_case__	:      Any     		=	unicodedata.normalize('''NFKD'''   ,__lowercase  )
										snake_case__	:      Dict     		=	''''''.join([c for c in outputs if not unicodedata.combining(__lowercase  )]  )
						if self.do_lower_case:
										snake_case__	:      str     		=	outputs.lower()

						return outputs



		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :str  ):
						snake_case__	:      Dict     		=	self.preprocess_text(__lowercase  )
						snake_case__	:      Any     		=	self.sp_model.encode(__lowercase   ,out_type=__lowercase  )
						snake_case__	:      List[str]     		=	[]
						for piece in pieces:
										if len(__lowercase  ) > 1 and piece[-1] == str(''','''  ) and piece[-2].isdigit():
														snake_case__	:      Union[str, Any]     		=	self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowercase   ,''''''  )  )
														if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																		if len(cur_pieces[0]  ) == 1:
																						snake_case__	:      Optional[Any]     		=	cur_pieces[1:]
																		else:
																						snake_case__	:      Optional[Any]     		=	cur_pieces[0][1:]
														cur_pieces.append(piece[-1]  )
														new_pieces.extend(__lowercase  )
										else:
														new_pieces.append(__lowercase  )

						return new_pieces



		def 	__lowerCamelCase			(     self     :Union[str, Any]   ,__lowercase     :Dict  ):
						return self.sp_model.PieceToId(__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple   ,__lowercase     :Optional[int]  ):
						return self.sp_model.IdToPiece(__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple   ,__lowercase     :List[Any]  ):
						snake_case__	:      Union[str, Any]     		=	''''''.join(__lowercase  ).replace(__lowercase   ,''' '''  ).strip()
						return out_string



		def 	__lowerCamelCase			(     self     :List[Any]   ,__lowercase     :List[int]   ,__lowercase     :Optional[List[int]] = None  ):
						snake_case__	:      Optional[Any]     		=	[self.sep_token_id]
						snake_case__	:      str     		=	[self.cls_token_id]
						if token_ids_a is None:
										return token_ids_a + sep + cls
						return token_ids_a + sep + token_ids_a + sep + cls



		def 	__lowerCamelCase			(     self     :str   ,__lowercase     :List[int]   ,__lowercase     :Optional[List[int]] = None   ,__lowercase     :bool = False  ):

						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=__lowercase   ,token_ids_a=__lowercase   ,already_has_special_tokens=__lowercase  )

						if token_ids_a is not None:
										return ([0] * len(__lowercase  )) + [1] + ([0] * len(__lowercase  )) + [1, 1]
						return ([0] * len(__lowercase  )) + [1, 1]



		def 	__lowerCamelCase			(     self     :List[str]   ,__lowercase     :List[int]   ,__lowercase     :Optional[List[int]] = None  ):
						snake_case__	:      List[Any]     		=	[self.sep_token_id]
						snake_case__	:      Optional[Any]     		=	[2]

						if token_ids_a is None:
										return len(token_ids_a + sep  ) * [0] + cls_segment_id
						return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id



		def 	__lowerCamelCase			(     self     :Optional[int]   ,__lowercase     :str   ,__lowercase     :Optional[str] = None  ):
						if not os.path.isdir(__lowercase  ):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
										return
						snake_case__	:      Union[str, Any]     		=	os.path.join(
						    __lowercase   ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

						if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowercase  ) and os.path.isfile(self.vocab_file  ):
										copyfile(self.vocab_file   ,__lowercase  )
						elif not os.path.isfile(self.vocab_file  ):
										with open(__lowercase   ,'''wb'''  ) as fi:
														snake_case__	:      str     		=	self.sp_model.serialized_model_proto()
														fi.write(__lowercase  )

						return (out_vocab_file,)






		def 	__lowerCamelCase			(     self     :Union[str, Any]   ,*__lowercase     :Optional[int]   ,**__lowercase     :Dict  ):
						snake_case__	:      Dict     		=	super()._decode(*__lowercase   ,**__lowercase  )
						snake_case__	:      List[Any]     		=	text.replace(''' '''   ,''''''  ).replace('''\u2582'''   ,''' '''  ).replace('''\u2583'''   ,'''\n'''  )
						return text

