






"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_   : int   			= '''▁'''

UpperCAmelCase_   : Optional[int]   			= {'''vocab_file''': '''spiece.model'''}

UpperCAmelCase_   : str   			= {
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

UpperCAmelCase_   : str   			= {
    '''google/pegasus-xsum''': 512,
}


UpperCAmelCase_   : Optional[Any]   			= logging.get_logger(__name__)




class 							lowerCAmelCase__				(			__a ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  ['''input_ids''', '''attention_mask''']
      def __init__(						self     :  Union[str, Any]						,	lowercase_     :  int						,	lowercase_     :  Any="<pad>"						,	lowercase_     :  Optional[int]="</s>"						,	lowercase_     :  Union[str, Any]="<unk>"						,	lowercase_     :  Optional[int]="<mask_2>"						,	lowercase_     :  Optional[int]="<mask_1>"						,	lowercase_     :  Dict=None						,	lowercase_     :  List[str]=103						,	lowercase_     :  Optional[Dict[str, Any]] = None						,	**lowercase_     :  List[Any]						,	):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		offset
          if additional_special_tokens is not None:
              if not isinstance(__a						,	__a):
                  raise TypeError(
                      F'additional_special_tokens should be of type {type(__a)}, but is'
                      F' {type(__a)}')

              SCREAMING_SNAKE_CASE_      :  Dict    						=		(
                  ([mask_token_sent] + additional_special_tokens)
                  if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                  else additional_special_tokens
              )
              # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
              additional_special_tokens_extended += [
                  F'<unk_{i}>' for i in range(len(__a)						,	self.offset - 1)
              ]

              if len(set(__a)) != len(__a):
                  raise ValueError(
                      '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                      F' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.')
              SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		additional_special_tokens_extended
          else:
              SCREAMING_SNAKE_CASE_      :  int    						=		[mask_token_sent] if mask_token_sent is not None else []
              additional_special_tokens += [F'<unk_{i}>' for i in range(2						,	self.offset)]

          SCREAMING_SNAKE_CASE_      :  str    						=		{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              eos_token=__a						,	unk_token=__a						,	mask_token=__a						,	pad_token=__a						,	mask_token_sent=__a						,	offset=__a						,	additional_special_tokens=__a						,	sp_model_kwargs=self.sp_model_kwargs						,	**__a						,	)
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		mask_token_sent
          SCREAMING_SNAKE_CASE_      :  Dict    						=		vocab_file
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.Load(__a)

          # add special tokens to encoder dict
          SCREAMING_SNAKE_CASE_      :  Dict[int, str]    						=		{
              0: self.pad_token,
              1: self.eos_token,
          }

          if self.mask_token_sent is not None:
              self.encoder.update(
                  {
                      2: self.mask_token_sent,
                      3: self.mask_token,
                  })

          if self.offset > 0:
              # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
              # mask_token_sent is already added to list -> so start at 1
              self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1						,	self.offset - 1)})

          SCREAMING_SNAKE_CASE_      :  Dict[str, int]    						=		{v: k for k, v in self.encoder.items()}
      @property
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          return len(self.sp_model) + self.offset
      def _SCREAMING_SNAKE_CASE (						self     :  int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Any    						=		{self.convert_ids_to_tokens(__a): i for i in range(self.vocab_size)}
          vocab.update(self.added_tokens_encoder)
          return vocab
      def __getstate__(						self     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		self.__dict__.copy()
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		None
          return state
      def __setstate__(						self     :  Tuple						,	lowercase_     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		d

          # for backward compatibility
          if not hasattr(self						,	'''sp_model_kwargs'''):
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		{}

          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.Load(self.vocab_file)
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  str):




          '''simple docstring'''


          return self.sp_model.encode(__a						,	out_type=__a)
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  str):




          '''simple docstring'''


          if token in self.decoder:
              return self.decoder[token]
          elif token in self.added_tokens_decoder:
              return self.added_tokens_decoder[token]
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		self.sp_model.piece_to_id(__a)
          return sp_id + self.offset
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  int):




          '''simple docstring'''


          if index in self.encoder:
              return self.encoder[index]
          elif index in self.added_tokens_encoder:
              return self.added_tokens_encoder[index]
          else:
              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		self.sp_model.IdToPiece(index - self.offset)
          return token
      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]						,	lowercase_     :  int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[]
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		""""""
          for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                  out_string += self.sp_model.decode(__a) + token
                  SCREAMING_SNAKE_CASE_      :  str    						=		[]
              else:
                  current_sub_tokens.append(__a)
          out_string += self.sp_model.decode(__a)
          return out_string.strip()
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  Optional[Any]=False):




          '''simple docstring'''


          return 1
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	lowercase_     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		set(self.all_special_ids)  # call it once instead of inside list comp
          all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

          return [1 if x in all_special_ids else 0 for x in seq]
      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]						,	lowercase_     :  List						,	lowercase_     :  Optional[List] = None						,	lowercase_     :  bool = False):




          '''simple docstring'''


          if already_has_special_tokens:
              return self._special_token_mask(__a)
          elif token_ids_a is None:
              return self._special_token_mask(__a) + [1]
          else:
              return self._special_token_mask(token_ids_a + token_ids_a) + [1]
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	lowercase_     :  Dict						,	lowercase_     :  Tuple=None):




          '''simple docstring'''


          if token_ids_a is None:
              return token_ids_a + [self.eos_token_id]
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return token_ids_a + token_ids_a + [self.eos_token_id]



      def _SCREAMING_SNAKE_CASE (						self     :  Tuple						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          if not os.path.isdir(__a):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory')
              return
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		os.path.join(
              __a						,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

          if os.path.abspath(self.vocab_file) != os.path.abspath(__a) and os.path.isfile(self.vocab_file):
              copyfile(self.vocab_file						,	__a)
          elif not os.path.isfile(self.vocab_file):
              with open(__a						,	'''wb''') as fi:
                  SCREAMING_SNAKE_CASE_      :  Any    						=		self.sp_model.serialized_model_proto()
                  fi.write(__a)

          return (out_vocab_file,)










import random
import unittest

import numpy as np

from diffusers import (
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionImgaImgPipeline,
    PNDMScheduler,
)
from diffusers.utils import floats_tensor
from diffusers.utils.testing_utils import (
    is_onnx_available,
    load_image,
    nightly,
    require_onnxruntime,
    require_torch_gpu,
)

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
	import onnxruntime as ort

class 						lowerCAmelCase   (       __a					,				unittest.TestCase	):

					'''simple docstring'''

					_A  :	List[Any]     =					'''hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline'''



					def 					lowerCAmelCase (     self :		int						,   __a :		List[Any]=0							)	->     Optional[int]:


												"""simple docstring"""

												__lowercase					:							Any			=   floats_tensor((1, 3, 128, 128)						,   rng=random.Random(__a							)							)
												__lowercase					:							Any			=   np.random.RandomState(__a							)
												__lowercase					:							Union[str, Any]			=   {
												    """prompt""": """A painting of a squirrel eating a burger""",
												    """image""": image,
												    """generator""": generator,
												    """num_inference_steps""": 3,
												    """strength""": 0.75,
												    """guidance_scale""": 7.5,
												    """output_type""": """numpy""",
												}
												return inputs



					def 					lowerCAmelCase (     self :		Union[str, Any]							)	->     Any:


												"""simple docstring"""

												__lowercase					:							List[Any]			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint						,   provider="""CPUExecutionProvider"""							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							Optional[Any]			=   self.get_dummy_inputs()
												__lowercase					:							str			=   pipe(**__a							).images
												__lowercase					:							Union[str, Any]			=   image[0, -3:, -3:, -1].flatten()

												assert image.shape == (1, 128, 128, 3)
												__lowercase					:							List[Any]			=   np.array([0.69643, 0.58484, 0.50314, 0.58760, 0.55368, 0.59643, 0.51529, 0.41217, 0.49087]							)
												assert np.abs(image_slice - expected_slice							).max() < 1E-1



					def 					lowerCAmelCase (     self :		Any							)	->     Any:


												"""simple docstring"""

												__lowercase					:							str			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint						,   provider="""CPUExecutionProvider"""							)
												__lowercase					:							Union[str, Any]			=   PNDMScheduler.from_config(pipe.scheduler.config						,   skip_prk_steps=__a							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							Dict			=   self.get_dummy_inputs()
												__lowercase					:							Optional[int]			=   pipe(**__a							).images
												__lowercase					:							Union[str, Any]			=   image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase					:							Any			=   np.array([0.61737, 0.54642, 0.53183, 0.54465, 0.52742, 0.60525, 0.49969, 0.40655, 0.48154]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def 					lowerCAmelCase (     self :		int							)	->     str:


												"""simple docstring"""

												__lowercase					:							int			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint						,   provider="""CPUExecutionProvider"""							)
												__lowercase					:							Union[str, Any]			=   LMSDiscreteScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												# warmup pass to apply optimizations
												__lowercase					:							Optional[int]			=   pipe(**self.get_dummy_inputs()							)

												__lowercase					:							Dict			=   self.get_dummy_inputs()
												__lowercase					:							str			=   pipe(**__a							).images
												__lowercase					:							Optional[Any]			=   image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase					:							Any			=   np.array([0.52761, 0.59977, 0.49033, 0.49619, 0.54282, 0.50311, 0.47600, 0.40918, 0.45203]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def 					lowerCAmelCase (     self :		Optional[int]							)	->     Optional[Any]:


												"""simple docstring"""

												__lowercase					:							List[str]			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint						,   provider="""CPUExecutionProvider"""							)
												__lowercase					:							List[Any]			=   EulerDiscreteScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							Any			=   self.get_dummy_inputs()
												__lowercase					:							int			=   pipe(**__a							).images
												__lowercase					:							Dict			=   image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase					:							Optional[int]			=   np.array([0.52911, 0.60004, 0.49229, 0.49805, 0.54502, 0.50680, 0.47777, 0.41028, 0.45304]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def 					lowerCAmelCase (     self :		List[Any]							)	->     str:


												"""simple docstring"""

												__lowercase					:							List[Any]			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint						,   provider="""CPUExecutionProvider"""							)
												__lowercase					:							Dict			=   EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							Tuple			=   self.get_dummy_inputs()
												__lowercase					:							Union[str, Any]			=   pipe(**__a							).images
												__lowercase					:							int			=   image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase					:							Union[str, Any]			=   np.array([0.52911, 0.60004, 0.49229, 0.49805, 0.54502, 0.50680, 0.47777, 0.41028, 0.45304]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def 					lowerCAmelCase (     self :		Dict							)	->     Union[str, Any]:


												"""simple docstring"""

												__lowercase					:							Optional[Any]			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint						,   provider="""CPUExecutionProvider"""							)
												__lowercase					:							Tuple			=   DPMSolverMultistepScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							str			=   self.get_dummy_inputs()
												__lowercase					:							Dict			=   pipe(**__a							).images
												__lowercase					:							int			=   image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase					:							Dict			=   np.array([0.65331, 0.58277, 0.48204, 0.56059, 0.53665, 0.56235, 0.50969, 0.40009, 0.46552]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1







@nightly
@require_onnxruntime
@require_torch_gpu
class 						lowerCAmelCase   (       unittest.TestCase	):

					'''simple docstring'''

					@property
					def 					lowerCAmelCase (     self :		Tuple							)	->     Optional[int]:


												"""simple docstring"""

												return (
												    "CUDAExecutionProvider",
												    {
												        "gpu_mem_limit": "15000000000",  # 15GB
												        "arena_extend_strategy": "kSameAsRequested",
												    },
												)



					@property
					def 					lowerCAmelCase (     self :		str							)	->     Tuple:


												"""simple docstring"""

												__lowercase					:							Union[str, Any]			=   ort.SessionOptions()
												__lowercase					:							Dict			=   False
												return options



					def 					lowerCAmelCase (     self :		int							)	->     Dict:


												"""simple docstring"""

												__lowercase					:							List[Any]			=   load_image(
												    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
												    """/img2img/sketch-mountains-input.jpg"""							)
												__lowercase					:							Union[str, Any]			=   init_image.resize((768, 512)							)
												# using the PNDM scheduler by default
												__lowercase					:							Optional[int]			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(
												    """CompVis/stable-diffusion-v1-4"""						,   revision="""onnx"""						,   safety_checker=__a						,   feature_extractor=__a						,   provider=self.gpu_provider						,   sess_options=self.gpu_options						,   )
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							int			=   """A fantasy landscape, trending on artstation"""

												__lowercase					:							int			=   np.random.RandomState(0							)
												__lowercase					:							List[str]			=   pipe(
												    prompt=__a						,   image=__a						,   strength=0.75						,   guidance_scale=7.5						,   num_inference_steps=10						,   generator=__a						,   output_type="""np"""						,   )
												__lowercase					:							Any			=   output.images
												__lowercase					:							Optional[int]			=   images[0, 255:258, 383:386, -1]

												assert images.shape == (1, 512, 768, 3)
												__lowercase					:							Optional[int]			=   np.array([0.4909, 0.5059, 0.5372, 0.4623, 0.4876, 0.5049, 0.4820, 0.4956, 0.5019]							)
												# TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 2E-2



					def 					lowerCAmelCase (     self :		List[Any]							)	->     Optional[Any]:


												"""simple docstring"""

												__lowercase					:							Union[str, Any]			=   load_image(
												    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
												    """/img2img/sketch-mountains-input.jpg"""							)
												__lowercase					:							Any			=   init_image.resize((768, 512)							)
												__lowercase					:							int			=   LMSDiscreteScheduler.from_pretrained(
												    """runwayml/stable-diffusion-v1-5"""						,   subfolder="""scheduler"""						,   revision="""onnx"""							)
												__lowercase					:							str			=   OnnxStableDiffusionImgaImgPipeline.from_pretrained(
												    """runwayml/stable-diffusion-v1-5"""						,   revision="""onnx"""						,   scheduler=__a						,   safety_checker=__a						,   feature_extractor=__a						,   provider=self.gpu_provider						,   sess_options=self.gpu_options						,   )
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase					:							Union[str, Any]			=   """A fantasy landscape, trending on artstation"""

												__lowercase					:							Any			=   np.random.RandomState(0							)
												__lowercase					:							Optional[Any]			=   pipe(
												    prompt=__a						,   image=__a						,   strength=0.75						,   guidance_scale=7.5						,   num_inference_steps=20						,   generator=__a						,   output_type="""np"""						,   )
												__lowercase					:							str			=   output.images
												__lowercase					:							Dict			=   images[0, 255:258, 383:386, -1]

												assert images.shape == (1, 512, 768, 3)
												__lowercase					:							Union[str, Any]			=   np.array([0.8043, 0.926, 0.9581, 0.8119, 0.8954, 0.913, 0.7209, 0.7463, 0.7431]							)
												# TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 2E-2