






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase	:							int				=logging.get_logger(__name__)

lowerCamelCase	:							Optional[Any]				='''▁'''

lowerCamelCase	:							Tuple				={'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

lowerCamelCase	:							int				={
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

lowerCamelCase	:							Optional[Any]				={
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

lowerCamelCase	:							int				={
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

lowerCamelCase	:							List[str]				={
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}





class 			__a  (    A__       ):
 _lowerCAmelCase       :       List[str]        = ["input_ids"]

 _lowerCAmelCase       :       int        = VOCAB_FILES_NAMES
 _lowerCAmelCase       :       Optional[Any]        = PRETRAINED_INIT_CONFIGURATION
 _lowerCAmelCase       :       str        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase       :       List[str]        = PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase       :       Any        = RESOURCE_FILES_NAMES







 def __init__(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   int							,   SCREAMING_SNAKE_CASE					:   Optional[int]=None							,   SCREAMING_SNAKE_CASE					:   Tuple=False							,   SCREAMING_SNAKE_CASE					:   Any="utf8"							,   SCREAMING_SNAKE_CASE					:   str="[UNK]"							,   SCREAMING_SNAKE_CASE					:   Tuple="[SEP]"							,   SCREAMING_SNAKE_CASE					:   str="[PAD]"							,   SCREAMING_SNAKE_CASE					:   List[str]="[CLS]"							,   SCREAMING_SNAKE_CASE					:   Tuple="[MASK]"							,   SCREAMING_SNAKE_CASE					:   Optional[Dict[str, Any]] = None							,   **SCREAMING_SNAKE_CASE					:   Union[str, Any]							,   ):


      '''simple docstring'''






      UpperCamelCase__			:   Tuple       				=  {} if sp_model_kwargs is None else sp_model_kwargs
      super().__init__(
          do_lower_case=SCREAMING_SNAKE_CASE							,   unk_token=SCREAMING_SNAKE_CASE							,   sep_token=SCREAMING_SNAKE_CASE							,   pad_token=SCREAMING_SNAKE_CASE							,   cls_token=SCREAMING_SNAKE_CASE							,   mask_token=SCREAMING_SNAKE_CASE							,   vocab_file=SCREAMING_SNAKE_CASE							,   encoding=SCREAMING_SNAKE_CASE							,   sp_model_kwargs=self.sp_model_kwargs							,   **SCREAMING_SNAKE_CASE							,   )
      UpperCamelCase__			:   Tuple       				=  do_lower_case
      UpperCamelCase__			:   Any       				=  sentencepiece_model_ckpt
      UpperCamelCase__			:   Union[str, Any]       				=  spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(SCREAMING_SNAKE_CASE						)

      # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
      if vocab_file is not None:
           UpperCamelCase__			:   str       				=  self.load_vocab(filepath=SCREAMING_SNAKE_CASE						)
      else:
           UpperCamelCase__			:   str       				=  {self.sp_model.id_to_piece(SCREAMING_SNAKE_CASE						): id for id in range(self.sp_model.get_piece_size()						)}
      UpperCamelCase__			:   Tuple       				=  {v: k for k, v in self.vocab.items()}







 def 					__lowercase					(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   Tuple						):


      '''simple docstring'''






      if text is None:
           return None

      UpperCamelCase__			:   int       				=  self.tokenize(SCREAMING_SNAKE_CASE						)
      UpperCamelCase__				,     UpperCamelCase__			:   List[Any]       				=  "", []

      for i, ch in enumerate(SCREAMING_SNAKE_CASE						):
           if ch in self.SP_CHAR_MAPPING:
                UpperCamelCase__			:   Union[str, Any]       				=  self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE						)
           else:
                UpperCamelCase__			:   Optional[int]       				=  unicodedata.normalize("NFKC"							,   SCREAMING_SNAKE_CASE						)
           if self.is_whitespace(SCREAMING_SNAKE_CASE						):
                continue
           normalized_text += ch
           char_mapping.extend([i] * len(SCREAMING_SNAKE_CASE						)						)

      UpperCamelCase__				,     UpperCamelCase__				,     UpperCamelCase__			:   Union[str, Any]       				=  normalized_text, [], 0

      if self.do_lower_case:
           UpperCamelCase__			:   str       				=  text.lower()

      for token in split_tokens:
           if token[:1] == "▁":
                UpperCamelCase__			:   str       				=  token[1:]
           UpperCamelCase__			:   List[Any]       				=  text[offset:].index(SCREAMING_SNAKE_CASE						) + offset
           UpperCamelCase__			:   Dict       				=  start + len(SCREAMING_SNAKE_CASE						)

           token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
           UpperCamelCase__			:   List[str]       				=  end
      return token_mapping







 @property
 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      return len(self.vocab						)







 def 					__lowercase					(		self					:   Union[str, Any]						):


      '''simple docstring'''






      return dict(self.vocab							,   **self.added_tokens_encoder						)







 def __getstate__(		self					:   List[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  self.__dict__.copy()
      UpperCamelCase__			:   List[Any]       				=  None
      return state







 def __setstate__(		self					:   List[str]							,   SCREAMING_SNAKE_CASE					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  d

      # for backward compatibility
      if not hasattr(self							,   "sp_model_kwargs"						):
           UpperCamelCase__			:   Optional[Any]       				=  {}

      UpperCamelCase__			:   Optional[int]       				=  spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(self.sentencepiece_model_ckpt						)







 def 					__lowercase					(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   Dict						):


      '''simple docstring'''






      return "".join((self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						) for c in text)						)







 def 					__lowercase					(		self					:   Optional[Any]							,   SCREAMING_SNAKE_CASE					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   Optional[int]=False							,   SCREAMING_SNAKE_CASE					:   int=64							,   SCREAMING_SNAKE_CASE					:   Tuple=0.1						):


      '''simple docstring'''






      if self.sp_model_kwargs.get("enable_sampling"						) is True:
           UpperCamelCase__			:   Dict       				=  True
      if self.sp_model_kwargs.get("alpha"						) is not None:
           UpperCamelCase__			:   Tuple       				=  self.sp_model_kwargs.get("alpha"						)
      if self.sp_model_kwargs.get("nbest_size"						) is not None:
           UpperCamelCase__			:   Tuple       				=  self.sp_model_kwargs.get("nbest_size"						)

      if not enable_sampling:
           UpperCamelCase__			:   Any       				=  self.sp_model.EncodeAsPieces(SCREAMING_SNAKE_CASE						)
      else:
           UpperCamelCase__			:   List[Any]       				=  self.sp_model.SampleEncodeAsPieces(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   List[str]       				=  []
      for pi, piece in enumerate(SCREAMING_SNAKE_CASE						):
           if piece == SPIECE_UNDERLINE:
                if not pieces[pi + 1].startswith(SCREAMING_SNAKE_CASE						) and pi != 0:
                     new_pieces.append(SCREAMING_SNAKE_CASE						)
                     continue
                else:
                     continue
           UpperCamelCase__			:   Dict       				=  0
           for i, chunk in enumerate(SCREAMING_SNAKE_CASE						):
                if chunk == SPIECE_UNDERLINE:
                     continue
                if self.is_ch_char(SCREAMING_SNAKE_CASE						) or self.is_punct(SCREAMING_SNAKE_CASE						):
                     if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                          new_pieces.append(piece[lst_i:i]						)
                     new_pieces.append(SCREAMING_SNAKE_CASE						)
                     UpperCamelCase__			:   Union[str, Any]       				=  i + 1
                elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                     if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                          new_pieces.append(piece[lst_i:i]						)
                     UpperCamelCase__			:   Union[str, Any]       				=  i
                elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                     if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                          new_pieces.append(piece[lst_i:i]						)
                     UpperCamelCase__			:   Any       				=  i
           if len(SCREAMING_SNAKE_CASE						) > lst_i:
                new_pieces.append(piece[lst_i:]						)
      return new_pieces







 def 					__lowercase					(		self					:   Optional[Any]							,   SCREAMING_SNAKE_CASE					:   List[str]						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  "".join(SCREAMING_SNAKE_CASE						).replace(SCREAMING_SNAKE_CASE							,   " "						).strip()
      return out_string







 def 					__lowercase					(		self					:   int							,   SCREAMING_SNAKE_CASE					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Union[str, Any]       				=  "".join(SCREAMING_SNAKE_CASE						).replace(SCREAMING_SNAKE_CASE							,   " "						).strip()
      return out_string







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Tuple						):


      '''simple docstring'''






      return self.vocab.get(SCREAMING_SNAKE_CASE							,   self.vocab.get(self.unk_token						)						)







 def 					__lowercase					(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   Optional[int]						):


      '''simple docstring'''






      return self.reverse_vocab.get(SCREAMING_SNAKE_CASE							,   self.unk_token						)







 def 					__lowercase					(		self					:   List[str]							,   SCREAMING_SNAKE_CASE					:   Any							,   SCREAMING_SNAKE_CASE					:   Optional[Any]=None						):


      '''simple docstring'''






      if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      UpperCamelCase__			:   str       				=  [self.cls_token_id]
      UpperCamelCase__			:   Union[str, Any]       				=  [self.sep_token_id]
      return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Any							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=None						):


      '''simple docstring'''






      if offset_mapping_a is None:
           return [(0, 0)] + offset_mapping_a + [(0, 0)]

      return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







 def 					__lowercase					(		self					:   Tuple							,   SCREAMING_SNAKE_CASE					:   int							,   SCREAMING_SNAKE_CASE					:   List[str]=None							,   SCREAMING_SNAKE_CASE					:   Any=False						):


      '''simple docstring'''






      if already_has_special_tokens:
           if token_ids_a is not None:
                raise ValueError(
                    "You should not supply a second sequence if the provided sequence of "
                    "ids is already formatted with special tokens for the model."						)
           return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is not None:
           return [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1]
      return [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1]







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   List[int]							,   SCREAMING_SNAKE_CASE					:   Optional[List[int]] = None						):


      '''simple docstring'''






      if token_ids_a is None:
           # [CLS] X [SEP]
           return (len(SCREAMING_SNAKE_CASE						) + 2) * [0]

      # [CLS] A [SEP] [SEP] B [SEP]
      return [0] * (len(SCREAMING_SNAKE_CASE						) + 1) + [1] * (len(SCREAMING_SNAKE_CASE						) + 3)







 def 					__lowercase					(		self					:   Union[str, Any]							,   SCREAMING_SNAKE_CASE					:   List[Any]						):


      '''simple docstring'''






      if "\u4e00" <= char <= "\u9fff":
           return True
      return False







 def 					__lowercase					(		self					:   Union[str, Any]							,   SCREAMING_SNAKE_CASE					:   Dict						):


      '''simple docstring'''






      if ("a" <= char <= "z") or ("A" <= char <= "Z"):
           return True
      return False







 def 					__lowercase					(		self					:   Any							,   SCREAMING_SNAKE_CASE					:   Any						):


      '''simple docstring'''






      if char in ",;:.?!~，；：。？！《》【】":
           return True
      return False







 def 					__lowercase					(		self					:   Optional[Any]							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]						):


      '''simple docstring'''






      if char == " " or char == "\t" or char == "\n" or char == "\r":
           return True
      if len(SCREAMING_SNAKE_CASE						) == 1:
           UpperCamelCase__			:   List[str]       				=  unicodedata.category(SCREAMING_SNAKE_CASE						)
           if cat == "Zs":
                return True
      return False







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  {}
      with io.open(SCREAMING_SNAKE_CASE							,   "r"							,   encoding="utf-8"						) as f:
           for index, line in enumerate(SCREAMING_SNAKE_CASE						):
                UpperCamelCase__			:   str       				=  line.rstrip("\n"						)
                UpperCamelCase__			:   str       				=  int(SCREAMING_SNAKE_CASE						)

      return token_to_idx







 def 					__lowercase					(		self					:   int							,   SCREAMING_SNAKE_CASE					:   str							,   SCREAMING_SNAKE_CASE					:   Optional[str] = None						):


      '''simple docstring'''






      UpperCamelCase__			:   List[str]       				=  0
      if os.path.isdir(SCREAMING_SNAKE_CASE						):
           UpperCamelCase__			:   Optional[Any]       				=  os.path.join(
               SCREAMING_SNAKE_CASE							,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
      else:
           UpperCamelCase__			:   Optional[int]       				=  (filename_prefix + "-" if filename_prefix else "") + save_directory
      with open(SCREAMING_SNAKE_CASE							,   "w"							,   encoding="utf-8"						) as writer:
           for token, token_index in sorted(self.vocab.items()							,   key=lambda SCREAMING_SNAKE_CASE						: kv[1]						):
                if index != token_index:
                     logger.warning(
                         F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                         " Please check that the vocabulary is not corrupted!"						)
                     UpperCamelCase__			:   Optional[int]       				=  token_index
                writer.write(token + "\n"						)
                index += 1

      UpperCamelCase__			:   Optional[int]       				=  os.path.join(SCREAMING_SNAKE_CASE							,   "sentencepiece.bpe.model"						)
      with open(SCREAMING_SNAKE_CASE							,   "wb"						) as fi:
           UpperCamelCase__			:   List[Any]       				=  self.sp_model.serialized_model_proto()
           fi.write(SCREAMING_SNAKE_CASE						)

      return (vocab_file,)







import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionImageVariationPipeline
from diffusers.utils.testing_utils import load_image, require_torch_gpu, slow, torch_device


lowerCamelCase	:							Tuple				=False





class 			__a  (    unittest.TestCase       ):
 pass







@slow
@require_torch_gpu
class 			__a  (    unittest.TestCase       ):







 def 					__lowercase					(		self					:   int						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[Any]       				=  VersatileDiffusionImageVariationPipeline.from_pretrained("shi-labs/versatile-diffusion"						)
      pipe.to(SCREAMING_SNAKE_CASE						)
      pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Union[str, Any]       				=  load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg"						)
      UpperCamelCase__			:   List[Any]       				=  torch.manual_seed(0						)
      UpperCamelCase__			:   List[Any]       				=  pipe(
          image=SCREAMING_SNAKE_CASE							,   generator=SCREAMING_SNAKE_CASE							,   guidance_scale=7.5							,   num_inference_steps=50							,   output_type="numpy"							,   ).images

      UpperCamelCase__			:   Union[str, Any]       				=  image[0, 2_53:2_56, 2_53:2_56, -1]

      assert image.shape == (1, 5_12, 5_12, 3)
      UpperCamelCase__			:   Optional[Any]       				=  np.array([0.0_4_4_1, 0.0_4_6_9, 0.0_5_0_7, 0.0_5_7_5, 0.0_6_3_2, 0.0_6_5_0, 0.0_8_6_5, 0.0_9_0_9, 0.0_9_4_5]						)

      assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2