




import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import XLMRobertaTokenizerFast

from diffusers import DDIMScheduler, KandinskyInpaintPipeline, KandinskyPriorPipeline, UNetaDConditionModel, VQModel
from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()
class 		UpperCAmelCase   (				UpperCAmelCase__       ,   unittest.TestCase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   KandinskyInpaintPipeline
						lowerCAmelCase_     		=   ['''prompt''', '''image_embeds''', '''negative_image_embeds''', '''image''', '''mask_image''']
						lowerCAmelCase_     		=   [
						    '''prompt''',
						    '''negative_prompt''',
						    '''image_embeds''',
						    '''negative_image_embeds''',
						    '''image''',
						    '''mask_image''',
						]
						lowerCAmelCase_     		=   [
						    '''generator''',
						    '''height''',
						    '''width''',
						    '''latents''',
						    '''guidance_scale''',
						    '''negative_prompt''',
						    '''num_inference_steps''',
						    '''return_dict''',
						    '''guidance_scale''',
						    '''num_images_per_prompt''',
						    '''output_type''',
						    '''return_dict''',
						]
						lowerCAmelCase_     		=   False






						@property
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							return 32






						@property
						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							return 32






						@property
						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							return self.time_input_dim






						@property
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							return self.time_input_dim * 4






						@property
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							return 1_00






						@property
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     XLMRobertaTokenizerFast.from_pretrained("YiYiXu/tiny-random-mclip-base"						)
							return tokenizer






						@property
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							torch.manual_seed(0						)
							snake_case_		    =     MCLIPConfig(
							    numDims=self.cross_attention_dim       ,	transformerDimensions=self.text_embedder_hidden_size       ,	hidden_size=self.text_embedder_hidden_size       ,	intermediate_size=37       ,	num_attention_heads=4       ,	num_hidden_layers=5       ,	vocab_size=10_05       ,	)

							snake_case_		    =     MultilingualCLIP(__lowercase						)
							snake_case_		    =     text_encoder.eval()

							return text_encoder






						@property
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							torch.manual_seed(0						)

							snake_case_		    =     {
							    "in_channels": 9,
							    # Out channels is double in channels because predicts mean and variance
							    "out_channels": 8,
							    "addition_embed_type": "text_image",
							    "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"),
							    "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"),
							    "mid_block_type": "UNetMidBlock2DSimpleCrossAttn",
							    "block_out_channels": (self.block_out_channels_a, self.block_out_channels_a * 2),
							    "layers_per_block": 1,
							    "encoder_hid_dim": self.text_embedder_hidden_size,
							    "encoder_hid_dim_type": "text_image_proj",
							    "cross_attention_dim": self.cross_attention_dim,
							    "attention_head_dim": 4,
							    "resnet_time_scale_shift": "scale_shift",
							    "class_embed_type": None,
							}

							snake_case_		    =     UNetaDConditionModel(**__lowercase						)
							return model






						@property
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							return {
							    "block_out_channels": [32, 64],
							    "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"],
							    "in_channels": 3,
							    "latent_channels": 4,
							    "layers_per_block": 1,
							    "norm_num_groups": 8,
							    "norm_type": "spatial",
							    "num_vq_embeddings": 12,
							    "out_channels": 3,
							    "up_block_types": [
							        "AttnUpDecoderBlock2D",
							        "UpDecoderBlock2D",
							    ],
							    "vq_embed_dim": 4,
							}






						@property
						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							torch.manual_seed(0						)
							snake_case_		    =     VQModel(**self.dummy_movq_kwargs						)
							return model






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     self.dummy_text_encoder
							snake_case_		    =     self.dummy_tokenizer
							snake_case_		    =     self.dummy_unet
							snake_case_		    =     self.dummy_movq

							snake_case_		    =     DDIMScheduler(
							    num_train_timesteps=10_00       ,	beta_schedule="linear"       ,	beta_start=0.0_0085       ,	beta_end=0.012       ,	clip_sample=__lowercase       ,	set_alpha_to_one=__lowercase       ,	steps_offset=1       ,	prediction_type="epsilon"       ,	thresholding=__lowercase       ,	)

							snake_case_		    =     {
							    "text_encoder": text_encoder,
							    "tokenizer": tokenizer,
							    "unet": unet,
							    "scheduler": scheduler,
							    "movq": movq,
							}

							return components






						def        snake_case__						(						self							:     str       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     List[Any]=0						):






							"""simple docstring"""






							snake_case_		    =     floats_tensor((1, self.cross_attention_dim)       ,	rng=random.Random(__lowercase						)						).to(__lowercase						)
							snake_case_		    =     floats_tensor((1, self.cross_attention_dim)       ,	rng=random.Random(seed + 1						)						).to(__lowercase						)
							# create init_image
							snake_case_		    =     floats_tensor((1, 3, 64, 64)       ,	rng=random.Random(__lowercase						)						).to(__lowercase						)
							snake_case_		    =     image.cpu().permute(0       ,	2       ,	3       ,	1						)[0]
							snake_case_		    =     Image.fromarray(np.uinta(__lowercase						)						).convert("RGB"						).resize((2_56, 2_56)						)
							# create mask
							snake_case_		    =     np.ones((64, 64)       ,	dtype=np.floataa						)
							snake_case_		    =     0

							if str(__lowercase						).startswith("mps"						):
								snake_case_		    =     torch.manual_seed(__lowercase						)
							else:
								snake_case_		    =     torch.Generator(device=__lowercase						).manual_seed(__lowercase						)
							snake_case_		    =     {
							    "prompt": "horse",
							    "image": init_image,
							    "mask_image": mask,
							    "image_embeds": image_embeds,
							    "negative_image_embeds": negative_image_embeds,
							    "generator": generator,
							    "height": 64,
							    "width": 64,
							    "num_inference_steps": 2,
							    "guidance_scale": 4.0,
							    "output_type": "np",
							}
							return inputs






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     "cpu"

							snake_case_		    =     self.get_dummy_components()

							snake_case_		    =     self.pipeline_class(**__lowercase						)
							snake_case_		    =     pipe.to(__lowercase						)

							pipe.set_progress_bar_config(disable=__lowercase						)

							snake_case_		    =     pipe(**self.get_dummy_inputs(__lowercase						)						)
							snake_case_		    =     output.images

							snake_case_		    =     pipe(
							    **self.get_dummy_inputs(__lowercase						)       ,	return_dict=__lowercase       ,	)[0]

							snake_case_		    =     image[0, -3:, -3:, -1]
							snake_case_		    =     image_from_tuple[0, -3:, -3:, -1]

							print(f"image.shape {image.shape}"						)

							assert image.shape == (1, 64, 64, 3)

							snake_case_		    =     np.array(
							    [0.832_6919, 0.7379_0467, 0.2091_8581, 0.930_9612, 0.551_1791, 0.4371_3328, 0.551_3321, 0.4992_2934, 0.5949_7786]						)

							assert (
							    np.abs(image_slice.flatten() - expected_slice						).max() < 1E-2
							), f" expected_slice {expected_slice}, but got {image_slice.flatten()}"
							assert (
							    np.abs(image_from_tuple_slice.flatten() - expected_slice						).max() < 1E-2
							), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}"






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							super().test_inference_batch_single_identical(expected_max_diff=3E-3						)







@slow
@require_torch_gpu
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							super().tearDown()
							gc.collect()
							torch.cuda.empty_cache()






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
							    "/kandinsky/kandinsky_inpaint_cat_with_hat_fp16.npy"						)

							snake_case_		    =     load_image(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png"						)
							snake_case_		    =     np.ones((7_68, 7_68)       ,	dtype=np.floataa						)
							snake_case_		    =     0

							snake_case_		    =     "a hat"

							snake_case_		    =     KandinskyPriorPipeline.from_pretrained(
							    "kandinsky-community/kandinsky-2-1-prior"       ,	torch_dtype=torch.floataa						)
							pipe_prior.to(__lowercase						)

							snake_case_		    =     KandinskyInpaintPipeline.from_pretrained(
							    "kandinsky-community/kandinsky-2-1-inpaint"       ,	torch_dtype=torch.floataa						)
							snake_case_		    =     pipeline.to(__lowercase						)
							pipeline.set_progress_bar_config(disable=__lowercase						)

							snake_case_		    =     torch.Generator(device="cpu"						).manual_seed(0						)
							snake_case_    ,      snake_case_		    =     pipe_prior(
							    __lowercase       ,	generator=__lowercase       ,	num_inference_steps=5       ,	negative_prompt=""       ,	).to_tuple()

							snake_case_		    =     pipeline(
							    __lowercase       ,	image=__lowercase       ,	mask_image=__lowercase       ,	image_embeds=__lowercase       ,	negative_image_embeds=__lowercase       ,	generator=__lowercase       ,	num_inference_steps=1_00       ,	height=7_68       ,	width=7_68       ,	output_type="np"       ,	)

							snake_case_		    =     output.images[0]

							assert image.shape == (7_68, 7_68, 3)

							assert_mean_pixel_difference(__lowercase       ,	__lowercase						)






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__				:     Optional[Any] 		=						logging.get_logger(__name__)

lowercase__				:     List[str] 		=						"▁"

lowercase__				:     Union[str, Any] 		=						{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowercase__				:     List[Any] 		=						{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowercase__				:     Tuple 		=						{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowercase__				:     Optional[int] 		=						{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

lowercase__				:     Dict 		=						{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}
class 		UpperCAmelCase   (				UpperCAmelCase__			):







						'''simple docstring'''





						lowerCAmelCase_     		=   ["input_ids"]

						lowerCAmelCase_     		=   VOCAB_FILES_NAMES
						lowerCAmelCase_     		=   PRETRAINED_INIT_CONFIGURATION
						lowerCAmelCase_     		=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase_     		=   PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase_     		=   RESOURCE_FILES_NAMES






						def __init__(						self							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Tuple=None       ,	__lowercase							:     List[str]=False       ,	__lowercase							:     List[str]="utf8"       ,	__lowercase							:     Union[str, Any]="[UNK]"       ,	__lowercase							:     List[str]="[SEP]"       ,	__lowercase							:     Optional[Any]="[PAD]"       ,	__lowercase							:     Any="[CLS]"       ,	__lowercase							:     Any="[MASK]"       ,	__lowercase							:     Optional[Dict[str, Any]] = None       ,	**__lowercase							:     Tuple       ,	):






							"""simple docstring"""






							snake_case_		    =     {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=__lowercase       ,	unk_token=__lowercase       ,	sep_token=__lowercase       ,	pad_token=__lowercase       ,	cls_token=__lowercase       ,	mask_token=__lowercase       ,	vocab_file=__lowercase       ,	encoding=__lowercase       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**__lowercase       ,	)
							snake_case_		    =     do_lower_case
							snake_case_		    =     sentencepiece_model_ckpt
							snake_case_		    =     spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(__lowercase						)

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
								snake_case_		    =     self.load_vocab(filepath=__lowercase						)
							else:
								snake_case_		    =     {self.sp_model.id_to_piece(__lowercase						): id for id in range(self.sp_model.get_piece_size()						)}
							snake_case_		    =     {v: k for k, v in self.vocab.items()}






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Optional[int]						):






							"""simple docstring"""






							if text is None:
								return None

							snake_case_		    =     self.tokenize(__lowercase						)
							snake_case_    ,      snake_case_		    =     "", []

							for i, ch in enumerate(__lowercase						):
								if ch in self.SP_CHAR_MAPPING:
									snake_case_		    =     self.SP_CHAR_MAPPING.get(__lowercase						)
								else:
									snake_case_		    =     unicodedata.normalize("NFKC"       ,	__lowercase						)
								if self.is_whitespace(__lowercase						):
									continue
								normalized_text += ch
								char_mapping.extend([i] * len(__lowercase						)						)

							snake_case_    ,      snake_case_    ,      snake_case_		    =     normalized_text, [], 0

							if self.do_lower_case:
								snake_case_		    =     text.lower()

							for token in split_tokens:
								if token[:1] == "▁":
									snake_case_		    =     token[1:]
								snake_case_		    =     text[offset:].index(__lowercase						) + offset
								snake_case_		    =     start + len(__lowercase						)

								token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
								snake_case_		    =     end
							return token_mapping






						@property
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							return len(self.vocab						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							return dict(self.vocab       ,	**self.added_tokens_encoder						)






						def __getstate__(						self							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     self.__dict__.copy()
							snake_case_		    =     None
							return state






						def __setstate__(						self							:     str       ,	__lowercase							:     str						):






							"""simple docstring"""






							snake_case_		    =     d

							# for backward compatibility
							if not hasattr(self       ,	"sp_model_kwargs"						):
								snake_case_		    =     {}

							snake_case_		    =     spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.sentencepiece_model_ckpt						)






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							return "".join((self.SP_CHAR_MAPPING.get(__lowercase       ,	__lowercase						) for c in text)						)






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     int       ,	__lowercase							:     Any=False       ,	__lowercase							:     str=64       ,	__lowercase							:     Optional[Any]=0.1						):






							"""simple docstring"""






							if self.sp_model_kwargs.get("enable_sampling"						) is True:
								snake_case_		    =     True
							if self.sp_model_kwargs.get("alpha"						) is not None:
								snake_case_		    =     self.sp_model_kwargs.get("alpha"						)
							if self.sp_model_kwargs.get("nbest_size"						) is not None:
								snake_case_		    =     self.sp_model_kwargs.get("nbest_size"						)

							if not enable_sampling:
								snake_case_		    =     self.sp_model.EncodeAsPieces(__lowercase						)
							else:
								snake_case_		    =     self.sp_model.SampleEncodeAsPieces(__lowercase       ,	__lowercase       ,	__lowercase						)
							snake_case_		    =     []
							for pi, piece in enumerate(__lowercase						):
								if piece == SPIECE_UNDERLINE:
									if not pieces[pi + 1].startswith(__lowercase						) and pi != 0:
										new_pieces.append(__lowercase						)
										continue
									else:
										continue
								snake_case_		    =     0
								for i, chunk in enumerate(__lowercase						):
									if chunk == SPIECE_UNDERLINE:
										continue
									if self.is_ch_char(__lowercase						) or self.is_punct(__lowercase						):
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										new_pieces.append(__lowercase						)
										snake_case_		    =     i + 1
									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										snake_case_		    =     i
									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										snake_case_		    =     i
								if len(__lowercase						) > lst_i:
									new_pieces.append(piece[lst_i:]						)
							return new_pieces






						def        snake_case__						(						self							:     List[Any]       ,	__lowercase							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     "".join(__lowercase						).replace(__lowercase       ,	" "						).strip()
							return out_string






						def        snake_case__						(						self							:     int       ,	__lowercase							:     int						):






							"""simple docstring"""






							snake_case_		    =     self.convert_ids_to_tokens(__lowercase						)
							snake_case_		    =     "".join(__lowercase						).replace(__lowercase       ,	" "						).strip()
							return out_string






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Any						):






							"""simple docstring"""






							return self.vocab.get(__lowercase       ,	self.vocab.get(self.unk_token						)						)






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[Any]						):






							"""simple docstring"""






							return self.reverse_vocab.get(__lowercase       ,	self.unk_token						)






						def        snake_case__						(						self							:     Optional[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     int=None						):






							"""simple docstring"""






							if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							snake_case_		    =     [self.cls_token_id]
							snake_case_		    =     [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[str]       ,	__lowercase							:     Any=None						):






							"""simple docstring"""






							if offset_mapping_a is None:
								return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     List[Any]=None       ,	__lowercase							:     Dict=False						):






							"""simple docstring"""






							if already_has_special_tokens:
								if token_ids_a is not None:
									raise ValueError(
									    "You should not supply a second sequence if the provided sequence of "
									    "ids is already formatted with special tokens for the model."						)
								return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
								return [1] + ([0] * len(__lowercase						)) + [1, 1] + ([0] * len(__lowercase						)) + [1]
							return [1] + ([0] * len(__lowercase						)) + [1]






						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     List[int]       ,	__lowercase							:     Optional[List[int]] = None						):






							"""simple docstring"""






							if token_ids_a is None:
								# [CLS] X [SEP]
								return (len(__lowercase						) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(__lowercase						) + 1) + [1] * (len(__lowercase						) + 3)






						def        snake_case__						(						self							:     Any       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							if "\u4e00" <= char <= "\u9fff":
								return True
							return False






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     Any						):






							"""simple docstring"""






							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
								return True
							return False






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Dict						):






							"""simple docstring"""






							if char in ",;:.?!~，；：。？！《》【】":
								return True
							return False






						def        snake_case__						(						self							:     Union[str, Any]       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							if char == " " or char == "\t" or char == "\n" or char == "\r":
								return True
							if len(__lowercase						) == 1:
								snake_case_		    =     unicodedata.category(__lowercase						)
								if cat == "Zs":
									return True
							return False






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     {}
							with io.open(__lowercase       ,	"r"       ,	encoding="utf-8"						) as f:
								for index, line in enumerate(__lowercase						):
									snake_case_		    =     line.rstrip("\n"						)
									snake_case_		    =     int(__lowercase						)

							return token_to_idx






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     str       ,	__lowercase							:     Optional[str] = None						):






							"""simple docstring"""






							snake_case_		    =     0
							if os.path.isdir(__lowercase						):
								snake_case_		    =     os.path.join(
								    __lowercase       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
							else:
								snake_case_		    =     (filename_prefix + "-" if filename_prefix else "") + save_directory
							with open(__lowercase       ,	"w"       ,	encoding="utf-8"						) as writer:
								for token, token_index in sorted(self.vocab.items()       ,	key=lambda __lowercase						: kv[1]						):
									if index != token_index:
										logger.warning(
										    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
										    " Please check that the vocabulary is not corrupted!"						)
										snake_case_		    =     token_index
									writer.write(token + "\n"						)
									index += 1

							snake_case_		    =     os.path.join(__lowercase       ,	"sentencepiece.bpe.model"						)
							with open(__lowercase       ,	"wb"						) as fi:
								snake_case_		    =     self.sp_model.serialized_model_proto()
								fi.write(__lowercase						)

							return (vocab_file,)
