
import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase	      = logging.get_logger(__name__)

UpperCAmelCase	      = '''▁'''

UpperCAmelCase	      = {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

UpperCAmelCase	      = {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

UpperCAmelCase	      = {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

UpperCAmelCase	      = {
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

UpperCAmelCase	      = {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	_UpperCamelCase     :     List[str]							 =       ["input_ids"]

	_UpperCamelCase     :     List[str]							 =       VOCAB_FILES_NAMES
	_UpperCamelCase     :     Tuple							 =       PRETRAINED_INIT_CONFIGURATION
	_UpperCamelCase     :     Any							 =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_UpperCamelCase     :     Optional[Any]							 =       PRETRAINED_VOCAB_FILES_MAP
	_UpperCamelCase     :     Optional[Any]							 =       RESOURCE_FILES_NAMES





	def __init__(       self		,    snake_case		,    snake_case=None		,    snake_case=False		,    snake_case="utf8"		,    snake_case="[UNK]"		,    snake_case="[SEP]"		,    snake_case="[PAD]"		,    snake_case="[CLS]"		,    snake_case="[MASK]"		,    snake_case = None		,    **snake_case		,    ):
		# Mask token behave like a normal word, i.e. include the space before it and
		# is included in the raw text, there should be a match in a non-normalized sentence.

		lowercase       =    {} if sp_model_kwargs is None else sp_model_kwargs
		super().__init__(
		    do_lower_case=snake_case		,    unk_token=snake_case		,    sep_token=snake_case		,    pad_token=snake_case		,    cls_token=snake_case		,    mask_token=snake_case		,    vocab_file=snake_case		,    encoding=snake_case		,    sp_model_kwargs=self.sp_model_kwargs		,    **snake_case		,    )
		lowercase       =    do_lower_case
		lowercase       =    sentencepiece_model_ckpt
		lowercase       =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
		self.sp_model.Load(snake_case       )

		# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
		if vocab_file is not None:
			lowercase       =    self.load_vocab(filepath=snake_case       )
		else:
			lowercase       =    {self.sp_model.id_to_piece(snake_case       ): id for id in range(self.sp_model.get_piece_size()       )}
		lowercase       =    {v: k for k, v in self.vocab.items()}





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		if text is None:
			return None

		lowercase       =    self.tokenize(snake_case       )
		lowercase  ,					lowercase       =    '', []

		for i, ch in enumerate(snake_case       ):
			if ch in self.SP_CHAR_MAPPING:
				lowercase       =    self.SP_CHAR_MAPPING.get(snake_case       )
			else:
				lowercase       =    unicodedata.normalize('NFKC'		,    snake_case       )
			if self.is_whitespace(snake_case       ):
				continue
			normalized_text += ch
			char_mapping.extend([i] * len(snake_case       )       )

		lowercase  ,					lowercase  ,					lowercase       =    normalized_text, [], 0

		if self.do_lower_case:
			lowercase       =    text.lower()

		for token in split_tokens:
			if token[:1] == "▁":
				lowercase       =    token[1:]
			lowercase       =    text[offset:].index(snake_case       ) + offset
			lowercase       =    start + len(snake_case       )

			token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)       )
			lowercase       =    end
		return token_mapping





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		return len(self.vocab       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		return dict(self.vocab		,    **self.added_tokens_encoder       )





	def __getstate__(       self       ):
		lowercase       =    self.__dict__.copy()
		lowercase       =    None
		return state





	def __setstate__(       self		,    snake_case       ):
		lowercase       =    d

		# for backward compatibility
		if not hasattr(self		,    'sp_model_kwargs'       ):
			lowercase       =    {}

		lowercase       =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
		self.sp_model.Load(self.sentencepiece_model_ckpt       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		return "".join((self.SP_CHAR_MAPPING.get(snake_case		,    snake_case       ) for c in text)       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case=False		,    snake_case=64		,    snake_case=0.1       ):

		if self.sp_model_kwargs.get('enable_sampling'       ) is True:
			lowercase       =    True
		if self.sp_model_kwargs.get('alpha'       ) is not None:
			lowercase       =    self.sp_model_kwargs.get('alpha'       )
		if self.sp_model_kwargs.get('nbest_size'       ) is not None:
			lowercase       =    self.sp_model_kwargs.get('nbest_size'       )

		if not enable_sampling:
			lowercase       =    self.sp_model.EncodeAsPieces(snake_case       )
		else:
			lowercase       =    self.sp_model.SampleEncodeAsPieces(snake_case		,    snake_case		,    snake_case       )
		lowercase       =    []
		for pi, piece in enumerate(snake_case       ):
			if piece == SPIECE_UNDERLINE:
				if not pieces[pi + 1].startswith(snake_case       ) and pi != 0:
					new_pieces.append(snake_case       )
					continue
				else:
					continue
			lowercase       =    0
			for i, chunk in enumerate(snake_case       ):
				if chunk == SPIECE_UNDERLINE:
					continue
				if self.is_ch_char(snake_case       ) or self.is_punct(snake_case       ):
					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
						new_pieces.append(piece[lst_i:i]       )
					new_pieces.append(snake_case       )
					lowercase       =    i + 1
				elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
						new_pieces.append(piece[lst_i:i]       )
					lowercase       =    i
				elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
						new_pieces.append(piece[lst_i:i]       )
					lowercase       =    i
			if len(snake_case       ) > lst_i:
				new_pieces.append(piece[lst_i:]       )
		return new_pieces





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase       =    ''.join(snake_case       ).replace(snake_case		,    ' '       ).strip()
		return out_string





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase       =    self.convert_ids_to_tokens(snake_case       )
		lowercase       =    ''.join(snake_case       ).replace(snake_case		,    ' '       ).strip()
		return out_string





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		return self.vocab.get(snake_case		,    self.vocab.get(self.unk_token       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		return self.reverse_vocab.get(snake_case		,    self.unk_token       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case=None       ):
		if token_ids_a is None:
			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
		lowercase       =    [self.cls_token_id]
		lowercase       =    [self.sep_token_id]
		return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case=None       ):
		if offset_mapping_a is None:
			return [(0, 0)] + offset_mapping_a + [(0, 0)]

		return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case=None		,    snake_case=False       ):

		if already_has_special_tokens:
			if token_ids_a is not None:
				raise ValueError(
				    'You should not supply a second sequence if the provided sequence of '
				    'ids is already formatted with special tokens for the model.'       )
			return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

		if token_ids_a is not None:
			return [1] + ([0] * len(snake_case       )) + [1, 1] + ([0] * len(snake_case       )) + [1]
		return [1] + ([0] * len(snake_case       )) + [1]





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case = None       ):
		# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
		if token_ids_a is None:
			# [CLS] X [SEP]
			return (len(snake_case       ) + 2) * [0]

		# [CLS] A [SEP] [SEP] B [SEP]
		return [0] * (len(snake_case       ) + 1) + [1] * (len(snake_case       ) + 3)





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		if "\u4e00" <= char <= "\u9fff":
			return True
		return False





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		if ("a" <= char <= "z") or ("A" <= char <= "Z"):
			return True
		return False





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		if char in ",;:.?!~，；：。？！《》【】":
			return True
		return False





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		if char == " " or char == "\t" or char == "\n" or char == "\r":
			return True
		if len(snake_case       ) == 1:
			lowercase       =    unicodedata.category(snake_case       )
			if cat == "Zs":
				return True
		return False





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase       =    {}
		with io.open(snake_case		,    'r'		,    encoding='utf-8'       ) as f:
			for index, line in enumerate(snake_case       ):
				lowercase       =    line.rstrip('\n'       )
				lowercase       =    int(snake_case       )

		return token_to_idx





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case = None       ):
		lowercase       =    0
		if os.path.isdir(snake_case       ):
			lowercase       =    os.path.join(
			    snake_case		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
		else:
			lowercase       =    (filename_prefix + '-' if filename_prefix else '') + save_directory
		with open(snake_case		,    'w'		,    encoding='utf-8'       ) as writer:
			for token, token_index in sorted(self.vocab.items()		,    key=lambda snake_case       : kv[1]       ):
				if index != token_index:
					logger.warning(
					    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
					    ' Please check that the vocabulary is not corrupted!'       )
					lowercase       =    token_index
				writer.write(token + '\n'       )
				index += 1

		lowercase       =    os.path.join(snake_case		,    'sentencepiece.bpe.model'       )
		with open(snake_case		,    'wb'       ) as fi:
			lowercase       =    self.sp_model.serialized_model_proto()
			fi.write(snake_case       )

		return (vocab_file,)



import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


UpperCAmelCase	      = logging.get_logger(__name__)






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    r'\w+[.]\d+'
	lowercase       =    re.findall(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							)
	for pat in pats:
		lowercase       =    key.replace(__SCREAMING_SNAKE_CASE ,							'_'.join(pat.split('.'							)							)							)
	return key






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							):
	lowercase       =    pt_tuple_key[:-1] + ('scale',)
	if (
	    any('norm' in str_ for str_ in pt_tuple_key							)
	    and (pt_tuple_key[-1] == "bias")
	    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
	    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
	):
		lowercase       =    pt_tuple_key[:-1] + ('scale',)
		return renamed_pt_tuple_key, pt_tensor
	elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
		lowercase       =    pt_tuple_key[:-1] + ('scale',)
		return renamed_pt_tuple_key, pt_tensor

	# embedding
	if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
		lowercase       =    pt_tuple_key[:-1] + ('embedding',)
		return renamed_pt_tuple_key, pt_tensor

	# conv layer
	lowercase       =    pt_tuple_key[:-1] + ('kernel',)
	if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
		lowercase       =    pt_tensor.transpose(2 ,							3 ,							1 ,							0							)
		return renamed_pt_tuple_key, pt_tensor

	# linear layer
	lowercase       =    pt_tuple_key[:-1] + ('kernel',)
	if pt_tuple_key[-1] == "weight":
		lowercase       =    pt_tensor.T
		return renamed_pt_tuple_key, pt_tensor

	# old PyTorch layer norm weight
	lowercase       =    pt_tuple_key[:-1] + ('weight',)
	if pt_tuple_key[-1] == "gamma":
		return renamed_pt_tuple_key, pt_tensor

	# old PyTorch layer norm bias
	lowercase       =    pt_tuple_key[:-1] + ('bias',)
	if pt_tuple_key[-1] == "beta":
		return renamed_pt_tuple_key, pt_tensor

	return pt_tuple_key, pt_tensor






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE=42							):
	# Step 1: Convert pytorch tensor to numpy
	lowercase       =    {k: v.numpy() for k, v in pt_state_dict.items()}

	# Step 2: Since the model is stateless, get random Flax params
	lowercase       =    flax_model.init_weights(PRNGKey(__SCREAMING_SNAKE_CASE							)							)

	lowercase       =    flatten_dict(__SCREAMING_SNAKE_CASE							)
	lowercase       =    {}

	# Need to change some parameters name to match Flax names
	for pt_key, pt_tensor in pt_state_dict.items():
		lowercase       =    rename_key(__SCREAMING_SNAKE_CASE							)
		lowercase       =    tuple(renamed_pt_key.split('.'							)							)

		# Correctly rename weight parameters
		lowercase  ,					lowercase       =    rename_key_and_reshape_tensor(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							)

		if flax_key in random_flax_state_dict:
			if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
				raise ValueError(
				    F'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
				    F'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''							)

        # also add unexpected weight so that warning is thrown
		lowercase       =    jnp.asarray(__SCREAMING_SNAKE_CASE							)

	return unflatten_dict(__SCREAMING_SNAKE_CASE							)

