code: 
  
  
  
  """simple docstring"""
  import os
  from shutil import copyfile
  from typing import TYPE_CHECKING, Optional, Tuple
  
  from tokenizers import processors
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  from ...utils.versions import require_version
  
  
  if TYPE_CHECKING:
  		from transformers.pipelines.conversational import Conversation
  
  require_version('tokenizers>=0.13.3')
  
  if is_sentencepiece_available():
  		from .tokenization_llama import LlamaTokenizer
  else:
  		_lowerCAmelCase					:Any  =      None
  
  _lowerCAmelCase					:Tuple  =      logging.get_logger(__name__)
  _lowerCAmelCase					:Optional[Any]  =      {'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}
  
  _lowerCAmelCase,_lowerCAmelCase					:Dict  =      '[INST]', '[/INST]'
  _lowerCAmelCase,_lowerCAmelCase					:List[str]  =      '<<SYS>>\n', '\n<</SYS>>\n\n'
  
  # fmt: off
  _lowerCAmelCase					:List[Any]  =      'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'
  class 							_UpperCAmelCase							(				a   ):
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             =VOCAB_FILES_NAMES
  					a__             =LlamaTokenizer
  					a__             ='''left'''
  					a__             =['''input_ids''', '''attention_mask''']
  
  
  
  
  
  					def __init__(      self ,			A=None ,			A=None ,			A=False ,			A="<unk>" ,			A="<s>" ,			A="</s>" ,			A=True ,			A=False ,			**A ,			)							->      int:
  									super().__init__(
  									    vocab_file=A ,			tokenizer_file=A ,			clean_up_tokenization_spaces=A ,			unk_token=A ,			bos_token=A ,			eos_token=A ,			**A ,			)
  									_UpperCAmelCase							:						List[Any]       							=					add_bos_token
  									_UpperCAmelCase							:						str       							=					add_eos_token
  									self.update_post_processor()
  
  									_UpperCAmelCase							:						List[str]       							=					vocab_file
  									_UpperCAmelCase							:						Optional[int]       							=					False if not self.vocab_file else True
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self	)							->      int:
  									_UpperCAmelCase							:						Tuple       							=					self.bos_token
  									_UpperCAmelCase							:						Optional[Any]       							=					self.bos_token_id
  
  									_UpperCAmelCase							:						Optional[int]       							=					self.eos_token
  									_UpperCAmelCase							:						str       							=					self.eos_token_id
  
  									_UpperCAmelCase							:						int       							=					f'{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'
  									_UpperCAmelCase							:						Any       							=					f'{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'
  
  									_UpperCAmelCase							:						Union[str, Any]       							=					[]
  									if self.add_bos_token:
  													special_tokens.append((bos, bos_token_id)	)
  									if self.add_eos_token:
  													special_tokens.append((eos, eos_token_id)	)
  									_UpperCAmelCase							:						Union[str, Any]       							=					processors.TemplateProcessing(
  									    single=A ,			pair=A ,			special_tokens=A	)
  
  
  
  
  
  					@property
  					def 				__lowerCAmelCase			(      self	)							->      int:
  									return self._add_eos_token
  
  
  
  
  
  					@property
  					def 				__lowerCAmelCase			(      self	)							->      Optional[int]:
  									return self._add_bos_token
  
  
  
  
  
  					@add_eos_token.setter
  					def 				__lowerCAmelCase			(      self ,			A	)							->      Union[str, Any]:
  									_UpperCAmelCase							:						int       							=					value
  									self.update_post_processor()
  
  
  
  
  
  					@add_bos_token.setter
  					def 				__lowerCAmelCase			(      self ,			A	)							->      int:
  									_UpperCAmelCase							:						Optional[int]       							=					value
  									self.update_post_processor()
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self ,			A ,			A = None	)							->      Tuple[str]:
  
  
  
  
  
  
  
  									if not self.can_save_slow_tokenizer:
  													raise ValueError(
  													    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
  													    '''tokenizer.'''	)
  
  									if not os.path.isdir(A	):
  													logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
  													return
  									_UpperCAmelCase							:						Tuple       							=					os.path.join(
  									    A ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
  
  									if os.path.abspath(self.vocab_file	) != os.path.abspath(A	):
  													copyfile(self.vocab_file ,			A	)
  
  									return (out_vocab_file,)
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self ,			A	)							->      int:
  									_UpperCAmelCase							:						int       							=					list(conversation.iter_texts()	)
  									if not all([is_user for is_user, msg in dialogue[::2]]	) or not all(
  									    [not is_user for is_user, msg in dialogue[1::2]]	):
  													raise ValueError(
  													    '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''	)
  
  									_UpperCAmelCase							:						List[Any]       							=					[]
  									if len(conversation.past_user_inputs	) > 0:
  													if not conversation.past_user_inputs[0].startswith(A	) or E_SYS not in conversation.past_user_inputs[0]:
  																	_UpperCAmelCase							:						Tuple       							=					(
  																	    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
  																	)
  									elif not dialogue[0][1].startswith(A	) or E_SYS not in dialogue[0][1]:
  													_UpperCAmelCase							:						Dict       							=					(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])
  
  									dialog_tokens += sum(
  									    [
  									        [self.bos_token_id]
  									        + self.encode(
  									            f'{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} ' ,			add_special_tokens=A	)
  									        + [self.eos_token_id]
  									        for prompt, answer in zip(dialogue[::2] ,			dialogue[1::2]	)
  									    ] ,			[] ,			)
  									if not (dialogue[-1][0]):
  													raise ValueError(f'Last message must be from user, got {dialogue[-1]["role"]}'	)
  									dialog_tokens += [self.bos_token_id] + self.encode(
  									    f'{B_INST} {(dialogue[-1][1]).strip()} {E_INST}' ,			add_special_tokens=A	)
  									return dialog_tokens
  
  
  
style_context: 
  
  
  
  """simple docstring"""
  import os
  from typing import List, Union
  
  import tensorflow as tf
  from tensorflow_text import BertTokenizer as BertTokenizerLayer
  from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs
  
  from .tokenization_bert import BertTokenizer
  class 							_UpperCAmelCase							(				tf.keras.layers.Layer   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A ,			A ,			A = None ,			A = None ,			A = None ,			A = "longest" ,			A = True ,			A = 5_1_2 ,			A = None ,			A = True ,			A = True ,			A = True ,			**A ,			)							->      Tuple:
           super().__init__()
           if use_fast_bert_tokenizer:
               _UpperCAmelCase							:						Union[str, Any]       							=					FastBertTokenizer(
                   A ,			token_out_type=tf.intaa ,			lower_case_nfd_strip_accents=A ,			**A	)
           else:
               _UpperCAmelCase							:						List[Any]       							=					tf.lookup.StaticVocabularyTable(
                   tf.lookup.KeyValueTensorInitializer(
                       keys=A ,			key_dtype=tf.string ,			values=tf.range(tf.size(A ,			out_type=tf.intaa	) ,			dtype=tf.intaa	) ,			value_dtype=tf.intaa ,			) ,			num_oov_buckets=1 ,			)
               _UpperCAmelCase							:						str       							=					BertTokenizerLayer(
                   A ,			token_out_type=tf.intaa ,			lower_case=A ,			**A	)
  
           _UpperCAmelCase							:						int       							=					vocab_list
           _UpperCAmelCase							:						Dict       							=					do_lower_case
           _UpperCAmelCase							:						Optional[int]       							=					cls_token_id or vocab_list.index('''[CLS]'''	)
           _UpperCAmelCase							:						Any       							=					sep_token_id or vocab_list.index('''[SEP]'''	)
           _UpperCAmelCase							:						List[Any]       							=					pad_token_id or vocab_list.index('''[PAD]'''	)
           _UpperCAmelCase							:						List[str]       							=					ShrinkLongestTrimmer(max_length - 3 ,			axis=1	)  # Allow room for special tokens
           _UpperCAmelCase							:						Tuple       							=					max_length
           _UpperCAmelCase							:						List[Any]       							=					padding
           _UpperCAmelCase							:						List[Any]       							=					truncation
           _UpperCAmelCase							:						Union[str, Any]       							=					pad_to_multiple_of
           _UpperCAmelCase							:						Union[str, Any]       							=					return_token_type_ids
           _UpperCAmelCase							:						Union[str, Any]       							=					return_attention_mask
  
  
  
  
  
       @classmethod
       def 				__lowerCAmelCase			(      cls ,			A ,			**A	)							->      Dict:  # noqa: F821
           _UpperCAmelCase							:						Optional[int]       							=					kwargs.pop('''do_lower_case''' ,			A	)
           _UpperCAmelCase							:						int       							=					tokenizer.do_lower_case if do_lower_case is None else do_lower_case
           _UpperCAmelCase							:						Optional[Any]       							=					kwargs.pop('''cls_token_id''' ,			A	)
           _UpperCAmelCase							:						str       							=					tokenizer.cls_token_id if cls_token_id is None else cls_token_id
           _UpperCAmelCase							:						str       							=					kwargs.pop('''sep_token_id''' ,			A	)
           _UpperCAmelCase							:						int       							=					tokenizer.sep_token_id if sep_token_id is None else sep_token_id
           _UpperCAmelCase							:						List[Any]       							=					kwargs.pop('''pad_token_id''' ,			A	)
           _UpperCAmelCase							:						Optional[Any]       							=					tokenizer.pad_token_id if pad_token_id is None else pad_token_id
  
           _UpperCAmelCase							:						int       							=					tokenizer.get_vocab()
           _UpperCAmelCase							:						List[str]       							=					sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()] ,			key=lambda A	: x[1]	)
           _UpperCAmelCase							:						Dict       							=					[entry[0] for entry in vocab]
           return cls(
               vocab_list=A ,			do_lower_case=A ,			cls_token_id=A ,			sep_token_id=A ,			pad_token_id=A ,			**A ,			)
  
  
  
  
  
       @classmethod
       def 				__lowerCAmelCase			(      cls ,			A ,			*A ,			**A	)							->      int:
           try:
               _UpperCAmelCase							:						int       							=					BertTokenizer.from_pretrained(A ,			*A ,			**A	)
           except:  # noqa: E722
               from .tokenization_bert_fast import BertTokenizerFast
  
               _UpperCAmelCase							:						Dict       							=					BertTokenizerFast.from_pretrained(A ,			*A ,			**A	)
           return cls.from_tokenizer(A ,			**A	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A	)							->      Any:
           if self.do_lower_case:
               _UpperCAmelCase							:						int       							=					case_fold_utfa(A	)
           _UpperCAmelCase							:						str       							=					self.tf_tokenizer.tokenize(A	)
           return tokens.merge_dims(1 ,			-1	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			)							->      Optional[Any]:
           if padding is None:
               _UpperCAmelCase							:						str       							=					self.padding
           if padding not in ("longest", "max_length"):
               raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''	)
           if max_length is not None and text_pair is not None:
               # Because we have to instantiate a Trimmer to do it properly
               raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''	)
           if max_length is None:
               _UpperCAmelCase							:						Union[str, Any]       							=					self.max_length
           if truncation is None:
               _UpperCAmelCase							:						List[Any]       							=					self.truncation
           if pad_to_multiple_of is None:
               _UpperCAmelCase							:						List[Any]       							=					self.pad_to_multiple_of
           if return_token_type_ids is None:
               _UpperCAmelCase							:						List[str]       							=					self.return_token_type_ids
           if return_attention_mask is None:
               _UpperCAmelCase							:						Any       							=					self.return_attention_mask
           if not isinstance(A ,			tf.Tensor	):
               _UpperCAmelCase							:						Dict       							=					tf.convert_to_tensor(A	)
           if text_pair is not None and not isinstance(A ,			tf.Tensor	):
               _UpperCAmelCase							:						str       							=					tf.convert_to_tensor(A	)
           if text_pair is not None:
               if text.shape.rank > 1:
                   raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''	)
               if text_pair.shape.rank > 1:
                   raise ValueError('''text_pair should not be multidimensional!'''	)
           if text.shape.rank == 2:
               _UpperCAmelCase							,       _UpperCAmelCase							:						str       							=					text[:, 0], text[:, 1]
           _UpperCAmelCase							:						Optional[Any]       							=					self.unpaired_tokenize(A	)
           if text_pair is None:  # Unpaired text
               if truncation:
                   _UpperCAmelCase							:						Optional[int]       							=					text[:, : max_length - 2]  # Allow room for special tokens
               _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					combine_segments(
                   (text,) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
           else:  # Paired text
               _UpperCAmelCase							:						Tuple       							=					self.unpaired_tokenize(A	)
               if truncation:
                   _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[int]       							=					self.paired_trimmer.trim([text, text_pair]	)
               _UpperCAmelCase							,       _UpperCAmelCase							:						Union[str, Any]       							=					combine_segments(
                   (text, text_pair) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
           if padding == "longest":
               _UpperCAmelCase							:						List[Any]       							=					input_ids.bounding_shape(axis=1	)
               if pad_to_multiple_of is not None:
                   # No ceiling division in tensorflow, so we negate floordiv instead
                   _UpperCAmelCase							:						Tuple       							=					pad_to_multiple_of * (-tf.math.floordiv(-pad_length ,			A	))
           else:
               _UpperCAmelCase							:						int       							=					max_length
  
           _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					pad_model_inputs(A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
           _UpperCAmelCase							:						Dict       							=					{'''input_ids''': input_ids}
           if return_attention_mask:
               _UpperCAmelCase							:						Optional[int]       							=					attention_mask
           if return_token_type_ids:
               _UpperCAmelCase							,       _UpperCAmelCase							:						Tuple       							=					pad_model_inputs(
                   A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
               _UpperCAmelCase							:						int       							=					token_type_ids
           return output
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      List[str]:
           return {
               "vocab_list": self.vocab_list,
               "do_lower_case": self.do_lower_case,
               "cls_token_id": self.cls_token_id,
               "sep_token_id": self.sep_token_id,
               "pad_token_id": self.pad_token_id,
           }
  
  
  