




import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.stable_diffusion_safe import StableDiffusionPipelineSafe as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import require_torch_gpu




class       __lowerCAmelCase				(							unittest.TestCase):






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         # clean up the VRAM after each test
         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    int ):
         lowercase			:Dict							=  1
         lowercase			:Any							=  3
         lowercase			:List[str]							=  (32, 32)

         lowercase			:List[str]							=  floats_tensor((batch_size, num_channels) + sizes				,    rng=random.Random(0 ) ).to(_lowerCAmelCase )
         return image






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         torch.manual_seed(0 )
         lowercase			:int							=  UNetaDConditionModel(
             block_out_channels=(32, 64)				,    layers_per_block=2				,    sample_size=32				,    in_channels=4				,    out_channels=4				,    down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")				,    up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")				,    cross_attention_dim=32				,    )
         return model






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         torch.manual_seed(0 )
         lowercase			:Tuple							=  AutoencoderKL(
             block_out_channels=[32, 64]				,    in_channels=3				,    out_channels=3				,    down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]				,    up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]				,    latent_channels=4				,    )
         return model






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         torch.manual_seed(0 )
         lowercase			:Any							=  CLIPTextConfig(
             bos_token_id=0				,    eos_token_id=2				,    hidden_size=32				,    intermediate_size=37				,    layer_norm_eps=1e-0_5				,    num_attention_heads=4				,    num_hidden_layers=5				,    pad_token_id=1				,    vocab_size=10_00				,    )
         return CLIPTextModel(_lowerCAmelCase )






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    int ):
         def extract(*_lowerCAmelCase:    Optional[int]				,    **_lowerCAmelCase:    Any ):




           class       __lowerCAmelCase				:






                  def __init__(      self:    Any ):
                    lowercase			:Tuple							=  torch.ones([0] )






                  def     SCREAMING_SNAKE_CASE			(      self:    Optional[int]				,    _lowerCAmelCase:    str ):
                    self.pixel_values.to(_lowerCAmelCase )
                    return self







           return Out()

         return extract






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         lowercase			:Optional[Any]							=  "cpu"  # ensure determinism for the device-dependent torch.Generator
         lowercase			:Optional[int]							=  self.dummy_cond_unet
         lowercase			:Union[str, Any]							=  DDIMScheduler(
             beta_start=0.0_00_85				,    beta_end=0.0_12				,    beta_schedule="scaled_linear"				,    clip_sample=_lowerCAmelCase				,    set_alpha_to_one=_lowerCAmelCase				,    )

         lowercase			:List[str]							=  self.dummy_vae
         lowercase			:int							=  self.dummy_text_encoder
         lowercase			:str							=  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip" )

         # make sure here that pndm scheduler skips prk
         lowercase			:List[str]							=  StableDiffusionPipeline(
             unet=_lowerCAmelCase				,    scheduler=_lowerCAmelCase				,    vae=_lowerCAmelCase				,    text_encoder=_lowerCAmelCase				,    tokenizer=_lowerCAmelCase				,    safety_checker=_lowerCAmelCase				,    feature_extractor=self.dummy_extractor				,    )
         lowercase			:str							=  sd_pipe.to(_lowerCAmelCase )
         sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase )

         lowercase			:Any							=  "A painting of a squirrel eating a burger"

         lowercase			:Union[str, Any]							=  torch.Generator(device=_lowerCAmelCase ).manual_seed(0 )
         lowercase			:Union[str, Any]							=  sd_pipe([prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=6.0				,    num_inference_steps=2				,    output_type="np" )
         lowercase			:Optional[int]							=  output.images

         lowercase			:Dict							=  torch.Generator(device=_lowerCAmelCase ).manual_seed(0 )
         lowercase			:Dict							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=6.0				,    num_inference_steps=2				,    output_type="np"				,    return_dict=_lowerCAmelCase				,    )[0]

         lowercase			:int							=  image[0, -3:, -3:, -1]
         lowercase			:List[str]							=  image_from_tuple[0, -3:, -3:, -1]

         assert image.shape == (1, 64, 64, 3)
         lowercase			:List[str]							=  np.array([0.57_56, 0.61_18, 0.50_05, 0.50_41, 0.54_71, 0.47_26, 0.49_76, 0.48_65, 0.48_64] )

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2
         assert np.abs(image_from_tuple_slice.flatten() - expected_slice ).max() < 1e-2






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         lowercase			:Optional[Any]							=  "cpu"  # ensure determinism for the device-dependent torch.Generator
         lowercase			:Optional[int]							=  self.dummy_cond_unet
         lowercase			:List[Any]							=  PNDMScheduler(skip_prk_steps=_lowerCAmelCase )
         lowercase			:Union[str, Any]							=  self.dummy_vae
         lowercase			:Union[str, Any]							=  self.dummy_text_encoder
         lowercase			:List[str]							=  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip" )

         # make sure here that pndm scheduler skips prk
         lowercase			:List[str]							=  StableDiffusionPipeline(
             unet=_lowerCAmelCase				,    scheduler=_lowerCAmelCase				,    vae=_lowerCAmelCase				,    text_encoder=_lowerCAmelCase				,    tokenizer=_lowerCAmelCase				,    safety_checker=_lowerCAmelCase				,    feature_extractor=self.dummy_extractor				,    )
         lowercase			:str							=  sd_pipe.to(_lowerCAmelCase )
         sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase )

         lowercase			:Any							=  "A painting of a squirrel eating a burger"
         lowercase			:Any							=  torch.Generator(device=_lowerCAmelCase ).manual_seed(0 )
         lowercase			:Any							=  sd_pipe([prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=6.0				,    num_inference_steps=2				,    output_type="np" )

         lowercase			:Optional[int]							=  output.images

         lowercase			:List[str]							=  torch.Generator(device=_lowerCAmelCase ).manual_seed(0 )
         lowercase			:Optional[int]							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=6.0				,    num_inference_steps=2				,    output_type="np"				,    return_dict=_lowerCAmelCase				,    )[0]

         lowercase			:List[str]							=  image[0, -3:, -3:, -1]
         lowercase			:Optional[int]							=  image_from_tuple[0, -3:, -3:, -1]

         assert image.shape == (1, 64, 64, 3)
         lowercase			:Union[str, Any]							=  np.array([0.51_25, 0.57_16, 0.48_28, 0.50_60, 0.56_50, 0.47_68, 0.51_85, 0.48_95, 0.49_93] )

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2
         assert np.abs(image_from_tuple_slice.flatten() - expected_slice ).max() < 1e-2






       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:int							=  StableDiffusionPipeline.from_pretrained(
             "hf-internal-testing/tiny-stable-diffusion-lms-pipe"				,    safety_checker=_lowerCAmelCase )
         assert isinstance(_lowerCAmelCase				,    _lowerCAmelCase )
         assert isinstance(pipe.scheduler				,    _lowerCAmelCase )
         assert pipe.safety_checker is None

         lowercase			:List[str]							=  pipe("example prompt"				,    num_inference_steps=2 ).images[0]
         assert image is not None

         # check that there's no error when saving a pipeline with one of the models being None
         with tempfile.TemporaryDirectory() as tmpdirname:
           pipe.save_pretrained(_lowerCAmelCase )
           lowercase			:List[Any]							=  StableDiffusionPipeline.from_pretrained(_lowerCAmelCase )

         # sanity check that the pipeline still works
         assert pipe.safety_checker is None
         lowercase			:List[Any]							=  pipe("example prompt"				,    num_inference_steps=2 ).images[0]
         assert image is not None






       @unittest.skipIf(torch_device != "cuda"				,    "This test requires a GPU" )
       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:List[Any]							=  self.dummy_cond_unet
         lowercase			:Any							=  PNDMScheduler(skip_prk_steps=_lowerCAmelCase )
         lowercase			:str							=  self.dummy_vae
         lowercase			:Any							=  self.dummy_text_encoder
         lowercase			:List[str]							=  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip" )

         # put models in fp16
         lowercase			:Optional[int]							=  unet.half()
         lowercase			:Optional[int]							=  vae.half()
         lowercase			:List[str]							=  bert.half()

         # make sure here that pndm scheduler skips prk
         lowercase			:int							=  StableDiffusionPipeline(
             unet=_lowerCAmelCase				,    scheduler=_lowerCAmelCase				,    vae=_lowerCAmelCase				,    text_encoder=_lowerCAmelCase				,    tokenizer=_lowerCAmelCase				,    safety_checker=_lowerCAmelCase				,    feature_extractor=self.dummy_extractor				,    )
         lowercase			:Optional[Any]							=  sd_pipe.to(_lowerCAmelCase )
         sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase )

         lowercase			:List[str]							=  "A painting of a squirrel eating a burger"
         lowercase			:Optional[int]							=  sd_pipe([prompt]				,    num_inference_steps=2				,    output_type="np" ).images

         assert image.shape == (1, 64, 64, 3)







@nightly
@require_torch_gpu
class       __lowerCAmelCase				(							unittest.TestCase):






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         # clean up the VRAM after each test
         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:Optional[Any]							=  StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"				,    safety_checker=_lowerCAmelCase )
         lowercase			:Dict							=  LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config )
         lowercase			:Tuple							=  sd_pipe.to(_lowerCAmelCase )
         sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase )

         lowercase			:List[str]							=  (
             "portrait of girl with smokey eyes makeup in abandoned hotel, grange clothes, redshift, wide high angle"
             " coloured polaroid photograph with flash, kodak film, hyper real, stunning moody cinematography, with"
             " anamorphic lenses, by maripol, fallen angels by wong kar - wai, style of suspiria and neon demon and"
             " children from bahnhof zoo, detailed "
         )
         lowercase			:str							=  40_03_66_03_46
         lowercase			:str							=  7

         # without safety guidance (sld_guidance_scale = 0)
         lowercase			:Tuple							=  torch.manual_seed(_lowerCAmelCase )
         lowercase			:Dict							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=_lowerCAmelCase				,    num_inference_steps=50				,    output_type="np"				,    width=5_12				,    height=5_12				,    sld_guidance_scale=0				,    )

         lowercase			:Any							=  output.images
         lowercase			:Any							=  image[0, -3:, -3:, -1]
         lowercase			:List[str]							=  [0.22_78, 0.22_31, 0.22_49, 0.23_33, 0.23_03, 0.18_85, 0.22_73, 0.21_44, 0.21_76]

         assert image.shape == (1, 5_12, 5_12, 3)

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2

         # without safety guidance (strong configuration)
         lowercase			:Any							=  torch.manual_seed(_lowerCAmelCase )
         lowercase			:List[Any]							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=_lowerCAmelCase				,    num_inference_steps=50				,    output_type="np"				,    width=5_12				,    height=5_12				,    sld_guidance_scale=20_00				,    sld_warmup_steps=7				,    sld_threshold=0.0_25				,    sld_momentum_scale=0.5				,    sld_mom_beta=0.7				,    )

         lowercase			:Optional[Any]							=  output.images
         lowercase			:str							=  image[0, -3:, -3:, -1]
         lowercase			:str							=  [0.23_83, 0.22_76, 0.2_36, 0.21_92, 0.21_86, 0.20_53, 0.19_71, 0.19_01, 0.17_19]

         assert image.shape == (1, 5_12, 5_12, 3)

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:List[str]							=  StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"				,    safety_checker=_lowerCAmelCase )
         lowercase			:int							=  LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config )
         lowercase			:Union[str, Any]							=  sd_pipe.to(_lowerCAmelCase )
         sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase )

         lowercase			:int							=  "padme amidala taking a bath artwork, safe for work, no nudity"
         lowercase			:Optional[int]							=  27_34_97_17_55
         lowercase			:str							=  7

         lowercase			:Optional[int]							=  torch.manual_seed(_lowerCAmelCase )
         lowercase			:List[str]							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=_lowerCAmelCase				,    num_inference_steps=50				,    output_type="np"				,    width=5_12				,    height=5_12				,    sld_guidance_scale=0				,    )

         lowercase			:Tuple							=  output.images
         lowercase			:Optional[Any]							=  image[0, -3:, -3:, -1]
         lowercase			:Optional[Any]							=  [0.35_02, 0.36_22, 0.33_96, 0.36_42, 0.34_78, 0.33_18, 0.35, 0.33_48, 0.32_97]

         assert image.shape == (1, 5_12, 5_12, 3)

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2

         lowercase			:int							=  torch.manual_seed(_lowerCAmelCase )
         lowercase			:Any							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=_lowerCAmelCase				,    num_inference_steps=50				,    output_type="np"				,    width=5_12				,    height=5_12				,    sld_guidance_scale=20_00				,    sld_warmup_steps=7				,    sld_threshold=0.0_25				,    sld_momentum_scale=0.5				,    sld_mom_beta=0.7				,    )

         lowercase			:List[Any]							=  output.images
         lowercase			:Any							=  image[0, -3:, -3:, -1]
         lowercase			:Dict							=  [0.55_31, 0.52_06, 0.48_95, 0.51_56, 0.51_82, 0.47_51, 0.48_02, 0.48_03, 0.44_43]

         assert image.shape == (1, 5_12, 5_12, 3)

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:Optional[Any]							=  StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5" )
         lowercase			:List[Any]							=  sd_pipe.to(_lowerCAmelCase )
         sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase )

         lowercase			:List[Any]							=  (
             "the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c."
             " leyendecker"
         )
         lowercase			:Optional[int]							=  10_44_35_52_34
         lowercase			:Optional[int]							=  12

         lowercase			:Tuple							=  torch.manual_seed(_lowerCAmelCase )
         lowercase			:Tuple							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=_lowerCAmelCase				,    num_inference_steps=50				,    output_type="np"				,    width=5_12				,    height=5_12				,    sld_guidance_scale=0				,    )

         lowercase			:Optional[int]							=  output.images
         lowercase			:Dict							=  image[0, -3:, -3:, -1]
         lowercase			:Optional[int]							=  np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] )

         assert image.shape == (1, 5_12, 5_12, 3)

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-7

         lowercase			:Optional[int]							=  torch.manual_seed(_lowerCAmelCase )
         lowercase			:Optional[int]							=  sd_pipe(
             [prompt]				,    generator=_lowerCAmelCase				,    guidance_scale=_lowerCAmelCase				,    num_inference_steps=50				,    output_type="np"				,    width=5_12				,    height=5_12				,    sld_guidance_scale=20_00				,    sld_warmup_steps=7				,    sld_threshold=0.0_25				,    sld_momentum_scale=0.5				,    sld_mom_beta=0.7				,    )

         lowercase			:Union[str, Any]							=  output.images
         lowercase			:Optional[int]							=  image[0, -3:, -3:, -1]
         lowercase			:Dict							=  np.array([0.58_18, 0.62_85, 0.68_35, 0.60_19, 0.6_25, 0.67_54, 0.60_96, 0.63_34, 0.65_61] )
         assert image.shape == (1, 5_12, 5_12, 3)

         assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2









import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase		:  Any	=   logging.get_logger(__name__)

_UpperCAmelCase		:  List[Any]	=   {"vocab_file": "sentencepiece.bpe.model"}

_UpperCAmelCase		:  List[str]	=   {
    "vocab_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
    }
}

_UpperCAmelCase		:  Any	=   {
    "camembert-base": 512,
}

_UpperCAmelCase		:  List[Any]	=   "▁"




class       __lowerCAmelCase				(							lowerCAmelCase):
       _a           =  VOCAB_FILES_NAMES
       _a           =  PRETRAINED_VOCAB_FILES_MAP
       _a           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a           =  ['''input_ids''', '''attention_mask''']






       def __init__(      self:    List[str]				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    Dict="<s>"				,    _lowerCAmelCase:    Union[str, Any]="</s>"				,    _lowerCAmelCase:    Optional[int]="</s>"				,    _lowerCAmelCase:    List[Any]="<s>"				,    _lowerCAmelCase:    Tuple="<unk>"				,    _lowerCAmelCase:    Union[str, Any]="<pad>"				,    _lowerCAmelCase:    str="<mask>"				,    _lowerCAmelCase:    int=["<s>NOTUSED", "</s>NOTUSED"]				,    _lowerCAmelCase:    Optional[Dict[str, Any]] = None				,    **_lowerCAmelCase:    Any				,    ):
         # Mask token behave like a normal word, i.e. include the space before it
         lowercase			:Dict							=  AddedToken(_lowerCAmelCase				,    lstrip=_lowerCAmelCase				,    rstrip=_lowerCAmelCase ) if isinstance(_lowerCAmelCase				,    _lowerCAmelCase ) else mask_token

         lowercase			:Optional[int]							=  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=_lowerCAmelCase				,    eos_token=_lowerCAmelCase				,    unk_token=_lowerCAmelCase				,    sep_token=_lowerCAmelCase				,    cls_token=_lowerCAmelCase				,    pad_token=_lowerCAmelCase				,    mask_token=_lowerCAmelCase				,    additional_special_tokens=_lowerCAmelCase				,    sp_model_kwargs=self.sp_model_kwargs				,    **_lowerCAmelCase				,    )
         lowercase			:Dict							=  spm.SentencePieceProcessor(**self.sp_model_kwargs )
         self.sp_model.Load(str(_lowerCAmelCase ) )
         lowercase			:Tuple							=  vocab_file
         # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
         # sentencepiece vocabulary (this is the case for <s> and </s>
         lowercase			:int							=  {"<s>NOTUSED": 0, "<pad>": 1, "</s>NOTUSED": 2, "<unk>": 3}
         lowercase			:Tuple							=  len(self.fairseq_tokens_to_ids )
         lowercase			:int							=  len(self.sp_model ) + len(self.fairseq_tokens_to_ids )
         lowercase			:Any							=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple				,    _lowerCAmelCase:    List[int]				,    _lowerCAmelCase:    Optional[List[int]] = None ):

         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         lowercase			:List[Any]							=  [self.cls_token_id]
         lowercase			:List[str]							=  [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep






       def     SCREAMING_SNAKE_CASE			(      self:    Dict				,    _lowerCAmelCase:    List[int]				,    _lowerCAmelCase:    Optional[List[int]] = None				,    _lowerCAmelCase:    bool = False ):
         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_lowerCAmelCase				,    token_ids_a=_lowerCAmelCase				,    already_has_special_tokens=_lowerCAmelCase )

         if token_ids_a is None:
           return [1] + ([0] * len(_lowerCAmelCase )) + [1]
         return [1] + ([0] * len(_lowerCAmelCase )) + [1, 1] + ([0] * len(_lowerCAmelCase )) + [1]






       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any]				,    _lowerCAmelCase:    List[int]				,    _lowerCAmelCase:    Optional[List[int]] = None ):
         lowercase			:Any							=  [self.sep_token_id]
         lowercase			:str							=  [self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






       @property
       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         return len(self.fairseq_tokens_to_ids ) + len(self.sp_model )






       def     SCREAMING_SNAKE_CASE			(      self:    Tuple ):
         lowercase			:Any							=  {self.convert_ids_to_tokens(_lowerCAmelCase ): i for i in range(self.vocab_size )}
         vocab.update(self.added_tokens_encoder )
         return vocab






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    _lowerCAmelCase:    str ):
         return self.sp_model.encode(_lowerCAmelCase				,    out_type=_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict				,    _lowerCAmelCase:    List[str] ):
         if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
         elif self.sp_model.PieceToId(_lowerCAmelCase ) == 0:
           # Convert sentence piece unk token to fairseq unk token index
           return self.unk_token_id
         return self.fairseq_offset + self.sp_model.PieceToId(_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    str				,    _lowerCAmelCase:    Tuple ):
         if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any]				,    _lowerCAmelCase:    List[str] ):
         lowercase			:Tuple							=  []
         lowercase			:Any							=  ""
         lowercase			:str							=  False
         for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
             if not prev_is_special:
               out_string += " "
             out_string += self.sp_model.decode(_lowerCAmelCase ) + token
             lowercase			:Optional[int]							=  True
             lowercase			:Any							=  []
           else:
             current_sub_tokens.append(_lowerCAmelCase )
             lowercase			:str							=  False
         out_string += self.sp_model.decode(_lowerCAmelCase )
         return out_string.strip()






       def __getstate__(      self:    Dict ):
         lowercase			:int							=  self.__dict__.copy()
         lowercase			:List[str]							=  None
         return state






       def __setstate__(      self:    Optional[Any]				,    _lowerCAmelCase:    Dict ):
         lowercase			:int							=  d

         # for backward compatibility
         if not hasattr(self				,    "sp_model_kwargs" ):
           lowercase			:Optional[int]							=  {}

         lowercase			:List[str]							=  spm.SentencePieceProcessor(**self.sp_model_kwargs )
         self.sp_model.Load(self.vocab_file )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int]				,    _lowerCAmelCase:    str				,    _lowerCAmelCase:    Optional[str] = None ):
         if not os.path.isdir(_lowerCAmelCase ):
           logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
           return
         lowercase			:Any							=  os.path.join(
             _lowerCAmelCase				,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

         if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowerCAmelCase ) and os.path.isfile(self.vocab_file ):
           copyfile(self.vocab_file				,    _lowerCAmelCase )
         elif not os.path.isfile(self.vocab_file ):
           with open(_lowerCAmelCase				,    "wb" ) as fi:
             lowercase			:Dict							=  self.sp_model.serialized_model_proto()
             fi.write(_lowerCAmelCase )

         return (out_vocab_file,)



