



"""simple docstring"""
import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
		from .tokenization_llama import LlamaTokenizer
else:
		_lowerCAmelCase					:Any  =      None

_lowerCAmelCase					:Tuple  =      logging.get_logger(__name__)
_lowerCAmelCase					:Optional[Any]  =      {'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

_lowerCAmelCase,_lowerCAmelCase					:Dict  =      '[INST]', '[/INST]'
_lowerCAmelCase,_lowerCAmelCase					:List[str]  =      '<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
_lowerCAmelCase					:List[Any]  =      'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'
class 							_UpperCAmelCase							(				a   ):





					'''simple docstring'''



					a__             =VOCAB_FILES_NAMES
					a__             =LlamaTokenizer
					a__             ='''left'''
					a__             =['''input_ids''', '''attention_mask''']





					def __init__(      self ,			A=None ,			A=None ,			A=False ,			A="<unk>" ,			A="<s>" ,			A="</s>" ,			A=True ,			A=False ,			**A ,			)							->      int:
									super().__init__(
									    vocab_file=A ,			tokenizer_file=A ,			clean_up_tokenization_spaces=A ,			unk_token=A ,			bos_token=A ,			eos_token=A ,			**A ,			)
									_UpperCAmelCase							:						List[Any]       							=					add_bos_token
									_UpperCAmelCase							:						str       							=					add_eos_token
									self.update_post_processor()

									_UpperCAmelCase							:						List[str]       							=					vocab_file
									_UpperCAmelCase							:						Optional[int]       							=					False if not self.vocab_file else True





					def 				__lowerCAmelCase			(      self	)							->      int:
									_UpperCAmelCase							:						Tuple       							=					self.bos_token
									_UpperCAmelCase							:						Optional[Any]       							=					self.bos_token_id

									_UpperCAmelCase							:						Optional[int]       							=					self.eos_token
									_UpperCAmelCase							:						str       							=					self.eos_token_id

									_UpperCAmelCase							:						int       							=					f'{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'
									_UpperCAmelCase							:						Any       							=					f'{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'

									_UpperCAmelCase							:						Union[str, Any]       							=					[]
									if self.add_bos_token:
													special_tokens.append((bos, bos_token_id)	)
									if self.add_eos_token:
													special_tokens.append((eos, eos_token_id)	)
									_UpperCAmelCase							:						Union[str, Any]       							=					processors.TemplateProcessing(
									    single=A ,			pair=A ,			special_tokens=A	)





					@property
					def 				__lowerCAmelCase			(      self	)							->      int:
									return self._add_eos_token





					@property
					def 				__lowerCAmelCase			(      self	)							->      Optional[int]:
									return self._add_bos_token





					@add_eos_token.setter
					def 				__lowerCAmelCase			(      self ,			A	)							->      Union[str, Any]:
									_UpperCAmelCase							:						int       							=					value
									self.update_post_processor()





					@add_bos_token.setter
					def 				__lowerCAmelCase			(      self ,			A	)							->      int:
									_UpperCAmelCase							:						Optional[int]       							=					value
									self.update_post_processor()





					def 				__lowerCAmelCase			(      self ,			A ,			A = None	)							->      Tuple[str]:







									if not self.can_save_slow_tokenizer:
													raise ValueError(
													    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
													    '''tokenizer.'''	)

									if not os.path.isdir(A	):
													logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
													return
									_UpperCAmelCase							:						Tuple       							=					os.path.join(
									    A ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

									if os.path.abspath(self.vocab_file	) != os.path.abspath(A	):
													copyfile(self.vocab_file ,			A	)

									return (out_vocab_file,)





					def 				__lowerCAmelCase			(      self ,			A	)							->      int:
									_UpperCAmelCase							:						int       							=					list(conversation.iter_texts()	)
									if not all([is_user for is_user, msg in dialogue[::2]]	) or not all(
									    [not is_user for is_user, msg in dialogue[1::2]]	):
													raise ValueError(
													    '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''	)

									_UpperCAmelCase							:						List[Any]       							=					[]
									if len(conversation.past_user_inputs	) > 0:
													if not conversation.past_user_inputs[0].startswith(A	) or E_SYS not in conversation.past_user_inputs[0]:
																	_UpperCAmelCase							:						Tuple       							=					(
																	    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
																	)
									elif not dialogue[0][1].startswith(A	) or E_SYS not in dialogue[0][1]:
													_UpperCAmelCase							:						Dict       							=					(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

									dialog_tokens += sum(
									    [
									        [self.bos_token_id]
									        + self.encode(
									            f'{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} ' ,			add_special_tokens=A	)
									        + [self.eos_token_id]
									        for prompt, answer in zip(dialogue[::2] ,			dialogue[1::2]	)
									    ] ,			[] ,			)
									if not (dialogue[-1][0]):
													raise ValueError(f'Last message must be from user, got {dialogue[-1]["role"]}'	)
									dialog_tokens += [self.bos_token_id] + self.encode(
									    f'{B_INST} {(dialogue[-1][1]).strip()} {E_INST}' ,			add_special_tokens=A	)
									return dialog_tokens







"""simple docstring"""
import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer
class 							_UpperCAmelCase							(				tf.keras.layers.Layer   ):





     '''simple docstring'''



     def __init__(      self ,			A ,			A ,			A = None ,			A = None ,			A = None ,			A = "longest" ,			A = True ,			A = 5_1_2 ,			A = None ,			A = True ,			A = True ,			A = True ,			**A ,			)							->      Tuple:
         super().__init__()
         if use_fast_bert_tokenizer:
             _UpperCAmelCase							:						Union[str, Any]       							=					FastBertTokenizer(
                 A ,			token_out_type=tf.intaa ,			lower_case_nfd_strip_accents=A ,			**A	)
         else:
             _UpperCAmelCase							:						List[Any]       							=					tf.lookup.StaticVocabularyTable(
                 tf.lookup.KeyValueTensorInitializer(
                     keys=A ,			key_dtype=tf.string ,			values=tf.range(tf.size(A ,			out_type=tf.intaa	) ,			dtype=tf.intaa	) ,			value_dtype=tf.intaa ,			) ,			num_oov_buckets=1 ,			)
             _UpperCAmelCase							:						str       							=					BertTokenizerLayer(
                 A ,			token_out_type=tf.intaa ,			lower_case=A ,			**A	)

         _UpperCAmelCase							:						int       							=					vocab_list
         _UpperCAmelCase							:						Dict       							=					do_lower_case
         _UpperCAmelCase							:						Optional[int]       							=					cls_token_id or vocab_list.index('''[CLS]'''	)
         _UpperCAmelCase							:						Any       							=					sep_token_id or vocab_list.index('''[SEP]'''	)
         _UpperCAmelCase							:						List[Any]       							=					pad_token_id or vocab_list.index('''[PAD]'''	)
         _UpperCAmelCase							:						List[str]       							=					ShrinkLongestTrimmer(max_length - 3 ,			axis=1	)  # Allow room for special tokens
         _UpperCAmelCase							:						Tuple       							=					max_length
         _UpperCAmelCase							:						List[Any]       							=					padding
         _UpperCAmelCase							:						List[Any]       							=					truncation
         _UpperCAmelCase							:						Union[str, Any]       							=					pad_to_multiple_of
         _UpperCAmelCase							:						Union[str, Any]       							=					return_token_type_ids
         _UpperCAmelCase							:						Union[str, Any]       							=					return_attention_mask





     @classmethod
     def 				__lowerCAmelCase			(      cls ,			A ,			**A	)							->      Dict:  # noqa: F821
         _UpperCAmelCase							:						Optional[int]       							=					kwargs.pop('''do_lower_case''' ,			A	)
         _UpperCAmelCase							:						int       							=					tokenizer.do_lower_case if do_lower_case is None else do_lower_case
         _UpperCAmelCase							:						Optional[Any]       							=					kwargs.pop('''cls_token_id''' ,			A	)
         _UpperCAmelCase							:						str       							=					tokenizer.cls_token_id if cls_token_id is None else cls_token_id
         _UpperCAmelCase							:						str       							=					kwargs.pop('''sep_token_id''' ,			A	)
         _UpperCAmelCase							:						int       							=					tokenizer.sep_token_id if sep_token_id is None else sep_token_id
         _UpperCAmelCase							:						List[Any]       							=					kwargs.pop('''pad_token_id''' ,			A	)
         _UpperCAmelCase							:						Optional[Any]       							=					tokenizer.pad_token_id if pad_token_id is None else pad_token_id

         _UpperCAmelCase							:						int       							=					tokenizer.get_vocab()
         _UpperCAmelCase							:						List[str]       							=					sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()] ,			key=lambda A	: x[1]	)
         _UpperCAmelCase							:						Dict       							=					[entry[0] for entry in vocab]
         return cls(
             vocab_list=A ,			do_lower_case=A ,			cls_token_id=A ,			sep_token_id=A ,			pad_token_id=A ,			**A ,			)





     @classmethod
     def 				__lowerCAmelCase			(      cls ,			A ,			*A ,			**A	)							->      int:
         try:
             _UpperCAmelCase							:						int       							=					BertTokenizer.from_pretrained(A ,			*A ,			**A	)
         except:  # noqa: E722
             from .tokenization_bert_fast import BertTokenizerFast

             _UpperCAmelCase							:						Dict       							=					BertTokenizerFast.from_pretrained(A ,			*A ,			**A	)
         return cls.from_tokenizer(A ,			**A	)





     def 				__lowerCAmelCase			(      self ,			A	)							->      Any:
         if self.do_lower_case:
             _UpperCAmelCase							:						int       							=					case_fold_utfa(A	)
         _UpperCAmelCase							:						str       							=					self.tf_tokenizer.tokenize(A	)
         return tokens.merge_dims(1 ,			-1	)





     def 				__lowerCAmelCase			(      self ,			A ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			)							->      Optional[Any]:
         if padding is None:
             _UpperCAmelCase							:						str       							=					self.padding
         if padding not in ("longest", "max_length"):
             raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''	)
         if max_length is not None and text_pair is not None:
             # Because we have to instantiate a Trimmer to do it properly
             raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''	)
         if max_length is None:
             _UpperCAmelCase							:						Union[str, Any]       							=					self.max_length
         if truncation is None:
             _UpperCAmelCase							:						List[Any]       							=					self.truncation
         if pad_to_multiple_of is None:
             _UpperCAmelCase							:						List[Any]       							=					self.pad_to_multiple_of
         if return_token_type_ids is None:
             _UpperCAmelCase							:						List[str]       							=					self.return_token_type_ids
         if return_attention_mask is None:
             _UpperCAmelCase							:						Any       							=					self.return_attention_mask
         if not isinstance(A ,			tf.Tensor	):
             _UpperCAmelCase							:						Dict       							=					tf.convert_to_tensor(A	)
         if text_pair is not None and not isinstance(A ,			tf.Tensor	):
             _UpperCAmelCase							:						str       							=					tf.convert_to_tensor(A	)
         if text_pair is not None:
             if text.shape.rank > 1:
                 raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''	)
             if text_pair.shape.rank > 1:
                 raise ValueError('''text_pair should not be multidimensional!'''	)
         if text.shape.rank == 2:
             _UpperCAmelCase							,       _UpperCAmelCase							:						str       							=					text[:, 0], text[:, 1]
         _UpperCAmelCase							:						Optional[Any]       							=					self.unpaired_tokenize(A	)
         if text_pair is None:  # Unpaired text
             if truncation:
                 _UpperCAmelCase							:						Optional[int]       							=					text[:, : max_length - 2]  # Allow room for special tokens
             _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					combine_segments(
                 (text,) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
         else:  # Paired text
             _UpperCAmelCase							:						Tuple       							=					self.unpaired_tokenize(A	)
             if truncation:
                 _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[int]       							=					self.paired_trimmer.trim([text, text_pair]	)
             _UpperCAmelCase							,       _UpperCAmelCase							:						Union[str, Any]       							=					combine_segments(
                 (text, text_pair) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
         if padding == "longest":
             _UpperCAmelCase							:						List[Any]       							=					input_ids.bounding_shape(axis=1	)
             if pad_to_multiple_of is not None:
                 # No ceiling division in tensorflow, so we negate floordiv instead
                 _UpperCAmelCase							:						Tuple       							=					pad_to_multiple_of * (-tf.math.floordiv(-pad_length ,			A	))
         else:
             _UpperCAmelCase							:						int       							=					max_length

         _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					pad_model_inputs(A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
         _UpperCAmelCase							:						Dict       							=					{'''input_ids''': input_ids}
         if return_attention_mask:
             _UpperCAmelCase							:						Optional[int]       							=					attention_mask
         if return_token_type_ids:
             _UpperCAmelCase							,       _UpperCAmelCase							:						Tuple       							=					pad_model_inputs(
                 A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
             _UpperCAmelCase							:						int       							=					token_type_ids
         return output





     def 				__lowerCAmelCase			(      self	)							->      List[str]:
         return {
             "vocab_list": self.vocab_list,
             "do_lower_case": self.do_lower_case,
             "cls_token_id": self.cls_token_id,
             "sep_token_id": self.sep_token_id,
             "pad_token_id": self.pad_token_id,
         }


