

'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
 import PIL


lowerCamelCase							:     Optional[int]								=			logging.get_logger(__name__)
class 						__lowerCAmelCase		(lowercase_  ):





      '''simple docstring'''






      lowerCAmelCase__      :  List[str]						      =							["""pixel_values"""]



      def __init__(self					:					Optional[int]     ,		UpperCamelCase					:					bool = True     ,		UpperCamelCase					:					Dict[str, int] = None     ,		UpperCamelCase					:					PILImageResampling = PIL.Image.NEAREST     ,		UpperCamelCase					:					bool = False     ,		UpperCamelCase					:					Dict[str, int] = None     ,		UpperCamelCase					:					Union[int, float] = 1 / 255     ,		UpperCamelCase					:					bool = False     ,		UpperCamelCase					:					bool = True     ,		UpperCamelCase					:					bool = True     ,		UpperCamelCase					:					Optional[Union[float, List[float]]] = None     ,		UpperCamelCase					:					Optional[Union[float, List[float]]] = None     ,		UpperCamelCase					:					bool = True     ,		**UpperCamelCase					:					Optional[int]     ,		):

             '''simple docstring'''







             super().__init__(**UpperCamelCase				)
             lowercase__       =				size if size is not None else {'''height''': 346, '''width''': 346}
             lowercase__       =				get_size_dict(UpperCamelCase				)
             lowercase__       =				crop_size if crop_size is not None else {'''height''': 289, '''width''': 289}
             lowercase__       =				get_size_dict(UpperCamelCase     ,		param_name='''crop_size'''				)

             lowercase__       =				do_resize
             lowercase__       =				size
             lowercase__       =				resample
             lowercase__       =				do_center_crop
             lowercase__       =				crop_size
             lowercase__       =				do_rescale
             lowercase__       =				rescale_factor
             lowercase__       =				rescale_offset
             lowercase__       =				do_normalize
             lowercase__       =				image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
             lowercase__       =				image_std if image_std is not None else IMAGENET_STANDARD_STD
             lowercase__       =				include_top



      def    UpperCamelCase__	(self					:					Dict     ,		UpperCamelCase					:					np.ndarray     ,		UpperCamelCase					:					Dict[str, int]     ,		UpperCamelCase					:					PILImageResampling = PIL.Image.NEAREST     ,		UpperCamelCase					:					Optional[Union[str, ChannelDimension]] = None     ,		**UpperCamelCase					:					Dict     ,		):

             '''simple docstring'''







             lowercase__       =				get_size_dict(UpperCamelCase				)
             if "height" not in size or "width" not in size:
                    raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"				)
             return resize(
                 UpperCamelCase     ,		size=(size['''height'''], size['''width'''])     ,		resample=UpperCamelCase     ,		data_format=UpperCamelCase     ,		**UpperCamelCase				)



      def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					np.ndarray     ,		UpperCamelCase					:					Dict[str, int]     ,		UpperCamelCase					:					Optional[Union[str, ChannelDimension]] = None     ,		**UpperCamelCase					:					Union[str, Any]     ,		):

             '''simple docstring'''







             lowercase__       =				get_size_dict(UpperCamelCase				)
             if "height" not in size or "width" not in size:
                    raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"				)
             return center_crop(UpperCamelCase     ,		size=(size['''height'''], size['''width'''])     ,		data_format=UpperCamelCase     ,		**UpperCamelCase				)



      def    UpperCamelCase__	(self					:					Tuple     ,		UpperCamelCase					:					np.ndarray     ,		UpperCamelCase					:					Union[int, float]     ,		UpperCamelCase					:					bool = True     ,		UpperCamelCase					:					Optional[Union[str, ChannelDimension]] = None     ,		**UpperCamelCase					:					List[str]     ,		):

             '''simple docstring'''







             if offset:
                    lowercase__       =				(image - 127.5) * scale
                    if data_format is not None:
                           lowercase__       =				to_channel_dimension_format(UpperCamelCase     ,		UpperCamelCase				)
                    lowercase__       =				rescaled_image.astype(np.floataa				)
             else:
                    lowercase__       =				rescale(UpperCamelCase     ,		scale=UpperCamelCase     ,		data_format=UpperCamelCase     ,		**UpperCamelCase				)
             return rescaled_image



      def    UpperCamelCase__	(self					:					List[Any]     ,		UpperCamelCase					:					np.ndarray     ,		UpperCamelCase					:					Union[float, List[float]]     ,		UpperCamelCase					:					Union[float, List[float]]     ,		UpperCamelCase					:					Optional[Union[str, ChannelDimension]] = None     ,		**UpperCamelCase					:					List[Any]     ,		):

             '''simple docstring'''







             return normalize(UpperCamelCase     ,		mean=UpperCamelCase     ,		std=UpperCamelCase     ,		data_format=UpperCamelCase     ,		**UpperCamelCase				)






      def    UpperCamelCase__	(self					:					Optional[int]     ,		UpperCamelCase					:					ImageInput     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					Dict[str, int] = None     ,		UpperCamelCase					:					Union[str, Any]=None     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					Dict[str, int] = None     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					float = None     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					Optional[Union[float, List[float]]] = None     ,		UpperCamelCase					:					Optional[Union[float, List[float]]] = None     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					Optional[Union[str, TensorType]] = None     ,		UpperCamelCase					:					ChannelDimension = ChannelDimension.FIRST     ,		**UpperCamelCase					:					List[Any]     ,		):

             '''simple docstring'''







             lowercase__       =				do_resize if do_resize is not None else self.do_resize
             lowercase__       =				resample if resample is not None else self.resample
             lowercase__       =				do_center_crop if do_center_crop is not None else self.do_center_crop
             lowercase__       =				do_rescale if do_rescale is not None else self.do_rescale
             lowercase__       =				rescale_factor if rescale_factor is not None else self.rescale_factor
             lowercase__       =				rescale_offset if rescale_offset is not None else self.rescale_offset
             lowercase__       =				do_normalize if do_normalize is not None else self.do_normalize
             lowercase__       =				image_mean if image_mean is not None else self.image_mean
             lowercase__       =				image_std if image_std is not None else self.image_std
             lowercase__       =				include_top if include_top is not None else self.include_top

             lowercase__       =				size if size is not None else self.size
             lowercase__       =				get_size_dict(UpperCamelCase				)
             lowercase__       =				crop_size if crop_size is not None else self.crop_size
             lowercase__       =				get_size_dict(UpperCamelCase     ,		param_name='''crop_size'''				)

             lowercase__       =				make_list_of_images(UpperCamelCase				)

             if not valid_images(UpperCamelCase				):
                    raise ValueError(
                        '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                        '''torch.Tensor, tf.Tensor or jax.ndarray.'''				)

             if do_resize and size is None or resample is None:
                    raise ValueError('''Size and resample must be specified if do_resize is True.'''				)

             if do_center_crop and crop_size is None:
                    raise ValueError('''Crop size must be specified if do_center_crop is True.'''				)

             if do_rescale and rescale_factor is None:
                    raise ValueError('''Rescale factor must be specified if do_rescale is True.'''				)

             if do_normalize and (image_mean is None or image_std is None):
                    raise ValueError('''Image mean and std must be specified if do_normalize is True.'''				)

             # All transformations expect numpy arrays.
             lowercase__       =				[to_numpy_array(UpperCamelCase				) for image in images]

             if do_resize:
                    lowercase__       =				[self.resize(image=UpperCamelCase     ,		size=UpperCamelCase     ,		resample=UpperCamelCase				) for image in images]

             if do_center_crop:
                    lowercase__       =				[self.center_crop(image=UpperCamelCase     ,		size=UpperCamelCase				) for image in images]

             if do_rescale:
                    lowercase__       =				[self.rescale(image=UpperCamelCase     ,		scale=UpperCamelCase     ,		offset=UpperCamelCase				) for image in images]

             if do_normalize:
                    lowercase__       =				[self.normalize(image=UpperCamelCase     ,		mean=UpperCamelCase     ,		std=UpperCamelCase				) for image in images]

             if include_top:
                    lowercase__       =				[self.normalize(image=UpperCamelCase     ,		mean=[0, 0, 0]     ,		std=UpperCamelCase				) for image in images]

             lowercase__       =				[to_channel_dimension_format(UpperCamelCase     ,		UpperCamelCase				) for image in images]

             lowercase__       =				{'''pixel_values''': images}
             return BatchFeature(data=UpperCamelCase     ,		tensor_type=UpperCamelCase				)








'''simple docstring'''

import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
 from .tokenization_llama import LlamaTokenizer
else:
 lowerCamelCase							:     Optional[int]								=			None

lowerCamelCase							:     List[Any]								=			logging.get_logger(__name__)
lowerCamelCase							:     Dict								=			{'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase		,						lowerCamelCase							:     str								=			'[INST]', '[/INST]'
lowerCamelCase		,						lowerCamelCase							:     Optional[Any]								=			'<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
lowerCamelCase							:     Optional[int]								=			'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'
class 						__lowerCAmelCase		(lowercase_  ):





      '''simple docstring'''






      lowerCAmelCase__      :  int						      =							VOCAB_FILES_NAMES
      lowerCAmelCase__      :  List[str]						      =							LlamaTokenizer
      lowerCAmelCase__      :  Tuple						      =							"""left"""
      lowerCAmelCase__      :  int						      =							["""input_ids""", """attention_mask"""]



      def __init__(self					:					Optional[Any]     ,		UpperCamelCase					:					Tuple=None     ,		UpperCamelCase					:					Optional[int]=None     ,		UpperCamelCase					:					List[str]=False     ,		UpperCamelCase					:					Dict="<unk>"     ,		UpperCamelCase					:					int="<s>"     ,		UpperCamelCase					:					Dict="</s>"     ,		UpperCamelCase					:					Any=True     ,		UpperCamelCase					:					Tuple=False     ,		**UpperCamelCase					:					List[str]     ,		):

             '''simple docstring'''







             super().__init__(
                 vocab_file=UpperCamelCase     ,		tokenizer_file=UpperCamelCase     ,		clean_up_tokenization_spaces=UpperCamelCase     ,		unk_token=UpperCamelCase     ,		bos_token=UpperCamelCase     ,		eos_token=UpperCamelCase     ,		**UpperCamelCase     ,		)
             lowercase__       =				add_bos_token
             lowercase__       =				add_eos_token
             self.update_post_processor()

             lowercase__       =				vocab_file
             lowercase__       =				False if not self.vocab_file else True



      def    UpperCamelCase__	(self					:					Tuple				):

             '''simple docstring'''







             lowercase__       =				self.bos_token
             lowercase__       =				self.bos_token_id

             lowercase__       =				self.eos_token
             lowercase__       =				self.eos_token_id

             lowercase__       =				f"{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"
             lowercase__       =				f"{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"

             lowercase__       =				[]
             if self.add_bos_token:
                    special_tokens.append((bos, bos_token_id)				)
             if self.add_eos_token:
                    special_tokens.append((eos, eos_token_id)				)
             lowercase__       =				processors.TemplateProcessing(
                 single=UpperCamelCase     ,		pair=UpperCamelCase     ,		special_tokens=UpperCamelCase				)



      @property
      def    UpperCamelCase__	(self					:					Union[str, Any]				):

             '''simple docstring'''







             return self._add_eos_token



      @property
      def    UpperCamelCase__	(self					:					Tuple				):

             '''simple docstring'''







             return self._add_bos_token



      @add_eos_token.setter
      def    UpperCamelCase__	(self					:					Union[str, Any]     ,		UpperCamelCase					:					int				):

             '''simple docstring'''







             lowercase__       =				value
             self.update_post_processor()



      @add_bos_token.setter
      def    UpperCamelCase__	(self					:					Optional[Any]     ,		UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             lowercase__       =				value
             self.update_post_processor()



      def    UpperCamelCase__	(self					:					Optional[Any]     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					Optional[str] = None				):

             '''simple docstring'''







             if not self.can_save_slow_tokenizer:
                    raise ValueError(
                        '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                        '''tokenizer.'''				)

             if not os.path.isdir(UpperCamelCase				):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
                    return
             lowercase__       =				os.path.join(
                 UpperCamelCase     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

             if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCamelCase				):
                    copyfile(self.vocab_file     ,		UpperCamelCase				)

             return (out_vocab_file,)






      def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					"Conversation"				):

             '''simple docstring'''







             lowercase__       =				list(conversation.iter_texts()				)
             if not all([is_user for is_user, msg in dialogue[::2]]				) or not all(
                 [not is_user for is_user, msg in dialogue[1::2]]				):
                    raise ValueError(
                        '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''				)

             lowercase__       =				[]
             if len(conversation.past_user_inputs				) > 0:
                    if not conversation.past_user_inputs[0].startswith(UpperCamelCase				) or E_SYS not in conversation.past_user_inputs[0]:
                           lowercase__       =				(
                               B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                           )
             elif not dialogue[0][1].startswith(UpperCamelCase				) or E_SYS not in dialogue[0][1]:
                    lowercase__       =				(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

             dialog_tokens += sum(
                 [
                     [self.bos_token_id]
                     + self.encode(
                         f"{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} "     ,		add_special_tokens=UpperCamelCase				)
                     + [self.eos_token_id]
                     for prompt, answer in zip(dialogue[::2]     ,		dialogue[1::2]				)
                 ]     ,		[]     ,		)
             if not (dialogue[-1][0]):
                    raise ValueError(f"Last message must be from user, got {dialogue[-1]['role']}"				)
             dialog_tokens += [self.bos_token_id] + self.encode(
                 f"{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"     ,		add_special_tokens=UpperCamelCase				)
             return dialog_tokens




