


'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase							:   Optional[int]  						=	logging.get_logger(__name__)

lowerCamelCase							:   Union[str, Any]  						=	{'vocab_file': 'spiece.model'}

lowerCamelCase							:   int  						=	{
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      def __init__(self    :    List[Any]   ,						UpperCamelCase    :    Optional[int]   ,						UpperCamelCase    :    List[str]=False   ,						UpperCamelCase    :    Tuple=True   ,						UpperCamelCase    :    Optional[int]=False   ,						UpperCamelCase    :    str="<s>"   ,						UpperCamelCase    :    Optional[Any]="</s>"   ,						UpperCamelCase    :    Any="<unk>"   ,						UpperCamelCase    :    Tuple="<sep>"   ,						UpperCamelCase    :    Union[str, Any]="<pad>"   ,						UpperCamelCase    :    Any="<cls>"   ,						UpperCamelCase    :    Tuple="<mask>"   ,						UpperCamelCase    :    Any=["<eop>", "<eod>"]   ,						UpperCamelCase    :    Optional[Dict[str, Any]] = None   ,						**UpperCamelCase    :    Optional[int]   ,						):

             '''simple docstring'''







             lowercase__      			= AddedToken(UpperCamelCase   ,						lstrip=UpperCamelCase   ,						rstrip=UpperCamelCase ) if isinstance(UpperCamelCase   ,						UpperCamelCase ) else mask_token

             lowercase__      			= {} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=UpperCamelCase   ,						remove_space=UpperCamelCase   ,						keep_accents=UpperCamelCase   ,						bos_token=UpperCamelCase   ,						eos_token=UpperCamelCase   ,						unk_token=UpperCamelCase   ,						sep_token=UpperCamelCase   ,						pad_token=UpperCamelCase   ,						cls_token=UpperCamelCase   ,						mask_token=UpperCamelCase   ,						additional_special_tokens=UpperCamelCase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**UpperCamelCase   ,						)

             lowercase__      			= 3

             lowercase__      			= do_lower_case
             lowercase__      			= remove_space
             lowercase__      			= keep_accents
             lowercase__      			= vocab_file

             lowercase__      			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
             self.sp_model.Load(UpperCamelCase )

             try:
                    import jieba
             except ModuleNotFoundError as error:
                    raise error.__class__(
                        '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
                        '''See https://pypi.org/project/jieba/ for installation.''' )
             lowercase__      			= jieba
             lowercase__      			= str.maketrans(''' \n'''   ,						'''\u2582\u2583''' )



      @property
      # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             return len(self.sp_model )



      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= {self.convert_ids_to_tokens(UpperCamelCase ): i for i in range(self.vocab_size )}
             vocab.update(self.added_tokens_encoder )
             return vocab



      def __getstate__(self    :    str ):

             '''simple docstring'''







             lowercase__      			= self.__dict__.copy()
             lowercase__      			= None
             return state



      def __setstate__(self    :    Dict   ,						UpperCamelCase    :    int ):

             '''simple docstring'''







             lowercase__      			= d

             # for backward compatibility
             if not hasattr(self   ,						'''sp_model_kwargs''' ):
                    lowercase__      			= {}

             lowercase__      			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
             self.sp_model.Load(self.vocab_file )



      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    Optional[Any] ):

             '''simple docstring'''







             if self.remove_space:
                    lowercase__      			= ''' '''.join(inputs.strip().split() )
             else:
                    lowercase__      			= inputs
             lowercase__      			= outputs.replace('''``'''   ,						'''"''' ).replace('''\'\''''   ,						'''"''' )

             if not self.keep_accents:
                    lowercase__      			= unicodedata.normalize('''NFKD'''   ,						UpperCamelCase )
                    lowercase__      			= ''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase )] )
             if self.do_lower_case:
                    lowercase__      			= outputs.lower()

             return outputs



      def       UpperCamelCase__		(self    :    List[str]   ,						UpperCamelCase    :    str ):

             '''simple docstring'''







             lowercase__      			= self.preprocess_text(UpperCamelCase )
             lowercase__      			= self.sp_model.encode(UpperCamelCase   ,						out_type=UpperCamelCase )
             lowercase__      			= []
             for piece in pieces:
                    if len(UpperCamelCase ) > 1 and piece[-1] == str(''',''' ) and piece[-2].isdigit():
                           lowercase__      			= self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase   ,						'''''' ) )
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0] ) == 1:
                                         lowercase__      			= cur_pieces[1:]
                                  else:
                                         lowercase__      			= cur_pieces[0][1:]
                           cur_pieces.append(piece[-1] )
                           new_pieces.extend(UpperCamelCase )
                    else:
                           new_pieces.append(UpperCamelCase )

             return new_pieces



      def       UpperCamelCase__		(self    :    Dict   ,						UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             return self.sp_model.PieceToId(UpperCamelCase )



      def       UpperCamelCase__		(self    :    int   ,						UpperCamelCase    :    Any ):

             '''simple docstring'''







             return self.sp_model.IdToPiece(UpperCamelCase )



      def       UpperCamelCase__		(self    :    List[str]   ,						UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= ''''''.join(UpperCamelCase ).replace(UpperCamelCase   ,						''' ''' ).strip()
             return out_string



      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None ):

             '''simple docstring'''







             lowercase__      			= [self.sep_token_id]
             lowercase__      			= [self.cls_token_id]
             if token_ids_a is None:
                    return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls



      def       UpperCamelCase__		(self    :    int   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None   ,						UpperCamelCase    :    bool = False ):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=UpperCamelCase   ,						token_ids_a=UpperCamelCase   ,						already_has_special_tokens=UpperCamelCase )

             if token_ids_a is not None:
                    return ([0] * len(UpperCamelCase )) + [1] + ([0] * len(UpperCamelCase )) + [1, 1]
             return ([0] * len(UpperCamelCase )) + [1, 1]



      def       UpperCamelCase__		(self    :    str   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None ):

             '''simple docstring'''







             lowercase__      			= [self.sep_token_id]
             lowercase__      			= [2]

             if token_ids_a is None:
                    return len(token_ids_a + sep ) * [0] + cls_segment_id
             return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id



      def       UpperCamelCase__		(self    :    Dict   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[str] = None ):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase ):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
                    return
             lowercase__      			= os.path.join(
                 UpperCamelCase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

             if os.path.abspath(self.vocab_file ) != os.path.abspath(UpperCamelCase ) and os.path.isfile(self.vocab_file ):
                    copyfile(self.vocab_file   ,						UpperCamelCase )
             elif not os.path.isfile(self.vocab_file ):
                    with open(UpperCamelCase   ,						'''wb''' ) as fi:
                           lowercase__      			= self.sp_model.serialized_model_proto()
                           fi.write(UpperCamelCase )

             return (out_vocab_file,)






      def       UpperCamelCase__		(self    :    Optional[Any]   ,						*UpperCamelCase    :    int   ,						**UpperCamelCase    :    List[str] ):

             '''simple docstring'''







             lowercase__      			= super()._decode(*UpperCamelCase   ,						**UpperCamelCase )
             lowercase__      			= text.replace(''' '''   ,						'''''' ).replace('''\u2582'''   ,						''' ''' ).replace('''\u2583'''   ,						'''\n''' )
             return text











'''simple docstring'''




import os
import tempfile
import unittest

import numpy as np

from diffusers.utils import is_flax_available
from diffusers.utils.testing_utils import require_flax, slow


if is_flax_available():
  import jax
  import jax.numpy as jnp
  from flax.jax_utils import replicate
  from flax.training.common_utils import shard

  from diffusers import FlaxDDIMScheduler, FlaxDiffusionPipeline, FlaxStableDiffusionPipeline


@require_flax
class  SCREAMING_SNAKE_CASE    (unittest.TestCase							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmpdirname:
               # pipeline has Flax weights
               __A						: Union[str, Any]      =       FlaxDiffusionPipeline.from_pretrained(
                   'hf-internal-testing/tiny-stable-diffusion-pipe'						,	safety_checker=_UpperCAmelCase						,	cache_dir=_UpperCAmelCase)

               __A						: Optional[Any]      =       [t[-1] for t in os.walk(os.path.join(_UpperCAmelCase						,	os.listdir(_UpperCAmelCase)[0]						,	'snapshots'))]
               __A						: int      =       [item for sublist in all_root_files for item in sublist]

               # None of the downloaded files should be a PyTorch file even if we have some here:
               # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_pytorch_model.bin
               assert not any(f.endswith('.bin') for f in files)




@slow
@require_flax
class  SCREAMING_SNAKE_CASE    (unittest.TestCase							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A ,__A						: Optional[int]      =       FlaxStableDiffusionPipeline.from_pretrained(
              'hf-internal-testing/tiny-stable-diffusion-pipe'						,	safety_checker=_UpperCAmelCase)

          __A						: Dict      =       (
              'A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of'
              ' field, close up, split lighting, cinematic'
          )

          __A						: Optional[Any]      =       jax.random.PRNGKey(0)
          __A						: int      =       4

          __A						: Tuple      =       jax.device_count()
          __A						: Union[str, Any]      =       num_samples * [prompt]
          __A						: Tuple      =       pipeline.prepare_inputs(_UpperCAmelCase)

          # shard inputs and rng
          __A						: str      =       replicate(_UpperCAmelCase)
          __A						: Tuple      =       jax.random.split(_UpperCAmelCase						,	_UpperCAmelCase)
          __A						: Union[str, Any]      =       shard(_UpperCAmelCase)

          __A						: Union[str, Any]      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images

          assert images.shape == (num_samples, 1, 64, 64, 3)
          if jax.device_count() == 8:
               assert np.abs(np.abs(images[0, 0, :2, :2, -2:]						,	dtype=np.floataa).sum() - 4.1514745) < 1e-3
               assert np.abs(np.abs(_UpperCAmelCase						,	dtype=np.floataa).sum() - 49947.875) < 5e-1

          __A						: List[str]      =       pipeline.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:])))
          assert len(_UpperCAmelCase) == num_samples


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A ,__A						: str      =       FlaxStableDiffusionPipeline.from_pretrained(
              'CompVis/stable-diffusion-v1-4'						,	revision='flax'						,	safety_checker=_UpperCAmelCase)

          __A						: List[Any]      =       (
              'A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of'
              ' field, close up, split lighting, cinematic'
          )

          __A						: Tuple      =       jax.random.PRNGKey(0)
          __A						: Any      =       50

          __A						: str      =       jax.device_count()
          __A						: Union[str, Any]      =       num_samples * [prompt]
          __A						: List[str]      =       pipeline.prepare_inputs(_UpperCAmelCase)

          # shard inputs and rng
          __A						: Dict      =       replicate(_UpperCAmelCase)
          __A						: Optional[Any]      =       jax.random.split(_UpperCAmelCase						,	_UpperCAmelCase)
          __A						: int      =       shard(_UpperCAmelCase)

          __A						: Tuple      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images

          assert images.shape == (num_samples, 1, 512, 512, 3)
          if jax.device_count() == 8:
               assert np.abs((np.abs(images[0, 0, :2, :2, -2:]						,	dtype=np.floataa).sum() - 0.05652401)) < 1e-3
               assert np.abs((np.abs(_UpperCAmelCase						,	dtype=np.floataa).sum() - 2383808.2)) < 5e-1


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A ,__A						: List[str]      =       FlaxStableDiffusionPipeline.from_pretrained(
              'CompVis/stable-diffusion-v1-4'						,	revision='bf16'						,	dtype=jnp.bfloataa						,	safety_checker=_UpperCAmelCase)

          __A						: List[Any]      =       (
              'A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of'
              ' field, close up, split lighting, cinematic'
          )

          __A						: str      =       jax.random.PRNGKey(0)
          __A						: Any      =       50

          __A						: Optional[int]      =       jax.device_count()
          __A						: int      =       num_samples * [prompt]
          __A						: Optional[int]      =       pipeline.prepare_inputs(_UpperCAmelCase)

          # shard inputs and rng
          __A						: Optional[int]      =       replicate(_UpperCAmelCase)
          __A						: List[str]      =       jax.random.split(_UpperCAmelCase						,	_UpperCAmelCase)
          __A						: Dict      =       shard(_UpperCAmelCase)

          __A						: str      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images

          assert images.shape == (num_samples, 1, 512, 512, 3)
          if jax.device_count() == 8:
               assert np.abs((np.abs(images[0, 0, :2, :2, -2:]						,	dtype=np.floataa).sum() - 0.04003906)) < 1e-3
               assert np.abs((np.abs(_UpperCAmelCase						,	dtype=np.floataa).sum() - 2373516.75)) < 5e-1


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A ,__A						: Optional[Any]      =       FlaxStableDiffusionPipeline.from_pretrained(
              'CompVis/stable-diffusion-v1-4'						,	revision='bf16'						,	dtype=jnp.bfloataa)

          __A						: Union[str, Any]      =       (
              'A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of'
              ' field, close up, split lighting, cinematic'
          )

          __A						: Any      =       jax.random.PRNGKey(0)
          __A						: List[str]      =       50

          __A						: Optional[int]      =       jax.device_count()
          __A						: List[Any]      =       num_samples * [prompt]
          __A						: List[Any]      =       pipeline.prepare_inputs(_UpperCAmelCase)

          # shard inputs and rng
          __A						: Union[str, Any]      =       replicate(_UpperCAmelCase)
          __A						: Optional[Any]      =       jax.random.split(_UpperCAmelCase						,	_UpperCAmelCase)
          __A						: List[str]      =       shard(_UpperCAmelCase)

          __A						: int      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images

          assert images.shape == (num_samples, 1, 512, 512, 3)
          if jax.device_count() == 8:
               assert np.abs((np.abs(images[0, 0, :2, :2, -2:]						,	dtype=np.floataa).sum() - 0.04003906)) < 1e-3
               assert np.abs((np.abs(_UpperCAmelCase						,	dtype=np.floataa).sum() - 2373516.75)) < 5e-1


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[Any]      =       FlaxDDIMScheduler(
              beta_start=0.00085						,	beta_end=0.012						,	beta_schedule='scaled_linear'						,	set_alpha_to_one=_UpperCAmelCase						,	steps_offset=1						,	)

          __A ,__A						: Any      =       FlaxStableDiffusionPipeline.from_pretrained(
              'CompVis/stable-diffusion-v1-4'						,	revision='bf16'						,	dtype=jnp.bfloataa						,	scheduler=_UpperCAmelCase						,	safety_checker=_UpperCAmelCase						,	)
          __A						: Optional[Any]      =       scheduler.create_state()

          __A						: Any      =       scheduler_state

          __A						: List[str]      =       (
              'A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of'
              ' field, close up, split lighting, cinematic'
          )

          __A						: Union[str, Any]      =       jax.random.PRNGKey(0)
          __A						: Optional[int]      =       50

          __A						: Optional[Any]      =       jax.device_count()
          __A						: Any      =       num_samples * [prompt]
          __A						: Optional[Any]      =       pipeline.prepare_inputs(_UpperCAmelCase)

          # shard inputs and rng
          __A						: int      =       replicate(_UpperCAmelCase)
          __A						: Any      =       jax.random.split(_UpperCAmelCase						,	_UpperCAmelCase)
          __A						: Tuple      =       shard(_UpperCAmelCase)

          __A						: Union[str, Any]      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images

          assert images.shape == (num_samples, 1, 512, 512, 3)
          if jax.device_count() == 8:
               assert np.abs((np.abs(images[0, 0, :2, :2, -2:]						,	dtype=np.floataa).sum() - 0.045043945)) < 1e-3
               assert np.abs((np.abs(_UpperCAmelCase						,	dtype=np.floataa).sum() - 2347693.5)) < 5e-1


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       (
              'A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of'
              ' field, close up, split lighting, cinematic'
          )

          __A						: int      =       jax.device_count()
          __A						: List[Any]      =       num_samples * [prompt]
          __A						: List[Any]      =       jax.random.split(jax.random.PRNGKey(0)						,	_UpperCAmelCase)

          __A ,__A						: Tuple      =       FlaxStableDiffusionPipeline.from_pretrained(
              'CompVis/stable-diffusion-v1-4'						,	revision='bf16'						,	dtype=jnp.bfloataa						,	safety_checker=_UpperCAmelCase						,	)

          __A						: str      =       replicate(_UpperCAmelCase)
          __A						: str      =       pipeline.prepare_inputs(_UpperCAmelCase)
          __A						: str      =       shard(_UpperCAmelCase)
          __A						: int      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images
          assert images.shape == (num_samples, 1, 512, 512, 3)
          __A						: Any      =       images[2, 0, 256, 10:17, 1]

          # With memory efficient attention
          __A ,__A						: str      =       FlaxStableDiffusionPipeline.from_pretrained(
              'CompVis/stable-diffusion-v1-4'						,	revision='bf16'						,	dtype=jnp.bfloataa						,	safety_checker=_UpperCAmelCase						,	use_memory_efficient_attention=_UpperCAmelCase						,	)

          __A						: Any      =       replicate(_UpperCAmelCase)
          __A						: List[Any]      =       pipeline.prepare_inputs(_UpperCAmelCase)
          __A						: Optional[Any]      =       shard(_UpperCAmelCase)
          __A						: List[Any]      =       pipeline(_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	jit=_UpperCAmelCase).images
          assert images_eff.shape == (num_samples, 1, 512, 512, 3)
          __A						: List[Any]      =       images[2, 0, 256, 10:17, 1]

          # I checked the results visually and they are very similar. However, I saw that the max diff is `1` and the `sum`
          # over the 8 images is exactly `256`, which is very suspicious. Testing a random slice for now.
          assert abs(slice_eff - slice).max() < 1e-2