code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import os
  from shutil import copyfile
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  if TYPE_CHECKING:
         from transformers.pipelines.conversational import Conversation
  
  lowercase__			=	logging.get_logger(__name__)
  
  lowercase__			=	{'''vocab_file''': '''tokenizer.model'''}
  
  lowercase__			=	{
      '''vocab_file''': {
          '''hf-internal-testing/llama-tokenizer''': '''https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model''',
      },
      '''tokenizer_file''': {
          '''hf-internal-testing/llama-tokenizer''': '''https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json''',
      },
  }
  lowercase__			=	{
      '''hf-internal-testing/llama-tokenizer''': 2_0_4_8,
  }
  lowercase__			=	'''▁'''
  
  lowercase__	,     lowercase__			=	'''[INST]''', '''[/INST]'''
  lowercase__	,     lowercase__			=	'''<<SYS>>\n''', '''\n<</SYS>>\n\n'''
  
  # fmt: off
  lowercase__			=	'''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
  answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
  that your responses are socially unbiased and positive in nature.
  
  If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
  correct. If you don\'t know the answer to a question, please don\'t share false information.'''
  
  
  
  
  
  
  class 						lowercase     (   UpperCAmelCase_):
        __lowercase 		=  VOCAB_FILES_NAMES
        __lowercase 		=  PRETRAINED_VOCAB_FILES_MAP
        __lowercase 		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __lowercase 		=  ['''input_ids''', '''attention_mask''']
  
  
  
  
  
  
        def __init__(self ,  a__ ,  a__="<unk>" ,  a__="<s>" ,  a__="</s>" ,  a__=None ,  a__ = None ,  a__=True ,  a__=False ,  a__=False ,  a__=True ,  **a__ ,  )       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				str       						= {} if sp_model_kwargs is None else sp_model_kwargs
           SCREAMING_SNAKE_CASE__			:				int       						= AddedToken(a__ ,  lstrip=a__ ,  rstrip=a__					) if isinstance(a__ ,  a__					) else bos_token
           SCREAMING_SNAKE_CASE__			:				Tuple       						= AddedToken(a__ ,  lstrip=a__ ,  rstrip=a__					) if isinstance(a__ ,  a__					) else eos_token
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= AddedToken(a__ ,  lstrip=a__ ,  rstrip=a__					) if isinstance(a__ ,  a__					) else unk_token
           SCREAMING_SNAKE_CASE__			:				str       						= AddedToken(a__ ,  lstrip=a__ ,  rstrip=a__					) if isinstance(a__ ,  a__					) else pad_token
           super().__init__(
               bos_token=a__ ,  eos_token=a__ ,  unk_token=a__ ,  pad_token=a__ ,  add_bos_token=a__ ,  add_eos_token=a__ ,  sp_model_kwargs=self.sp_model_kwargs ,  clean_up_tokenization_spaces=a__ ,  legacy=a__ ,  **a__ ,  )
           if legacy:
              logger.warning_once(
                  f"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
                  ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''					)
           SCREAMING_SNAKE_CASE__			:				Tuple       						= legacy
           SCREAMING_SNAKE_CASE__			:				Tuple       						= vocab_file
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= add_bos_token
           SCREAMING_SNAKE_CASE__			:				int       						= add_eos_token
           SCREAMING_SNAKE_CASE__			:				Tuple       						= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
           self.sp_model.Load(a__					)
  
  
  
  
  
  
        def __getstate__(self					)       ->   int:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= self.__dict__.copy()
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= None
           SCREAMING_SNAKE_CASE__			:				Any       						= self.sp_model.serialized_model_proto()
           return state
  
  
  
  
  
  
        def __setstate__(self ,  a__					)       ->   List[str]:
           SCREAMING_SNAKE_CASE__			:				str       						= d
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
           self.sp_model.LoadFromSerializedProto(self.sp_model_proto					)
  
  
  
  
  
  
        @property
        def    A__(self					)       ->   str:
           return self.sp_model.get_piece_size()
  
  
  
  
  
  
        def    A__(self					)       ->   Optional[int]:
           SCREAMING_SNAKE_CASE__			:				str       						= {self.convert_ids_to_tokens(a__					): i for i in range(self.vocab_size					)}
           vocab.update(self.added_tokens_encoder					)
           return vocab
  
  
  
  
  
  
        def    A__(self ,  a__ ,  **a__					)       ->   List[str]:
           # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
           # the beginning of the text
           if not self.legacy:
              SCREAMING_SNAKE_CASE__			:				List[str]       						= SPIECE_UNDERLINE + text.replace(a__ ,  ''' '''					)
           return super().tokenize(a__ ,  **a__					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   Any:
           if not self.legacy:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= text.startswith(a__					)
              if is_first:
                 SCREAMING_SNAKE_CASE__			:				str       						= text[1:]
  
           SCREAMING_SNAKE_CASE__			:				str       						= self.sp_model.encode(a__ ,  out_type=a__					)
  
           if not self.legacy and not is_first and not text.startswith(''' '''					) and tokens[0].startswith(a__					):
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= ([tokens[0][1:]] if len(tokens[0]					) > 1 else []) + tokens[1:]
           return tokens
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   List[Any]:
           return self.sp_model.piece_to_id(a__					)
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   List[Any]:
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= self.sp_model.IdToPiece(a__					)
           return token
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   Optional[int]:
           SCREAMING_SNAKE_CASE__			:				str       						= []
           SCREAMING_SNAKE_CASE__			:				Any       						= ''''''
           SCREAMING_SNAKE_CASE__			:				Any       						= False
           for i, token in enumerate(a__					):
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                 if not prev_is_special and i != 0:
                    out_string += " "
                 out_string += self.sp_model.decode(a__					) + token
                 SCREAMING_SNAKE_CASE__			:				str       						= True
                 SCREAMING_SNAKE_CASE__			:				Tuple       						= []
              else:
                 current_sub_tokens.append(a__					)
                 SCREAMING_SNAKE_CASE__			:				int       						= False
           out_string += self.sp_model.decode(a__					)
           return out_string
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   Tuple[str]:
           if not os.path.isdir(a__					):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
              return
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= os.path.join(
               a__ ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
  
           if os.path.abspath(self.vocab_file					) != os.path.abspath(a__					) and os.path.isfile(self.vocab_file					):
              copyfile(self.vocab_file ,  a__					)
           elif not os.path.isfile(self.vocab_file					):
              with open(a__ ,  '''wb'''					) as fi:
                 SCREAMING_SNAKE_CASE__			:				Any       						= self.sp_model.serialized_model_proto()
                 fi.write(a__					)
  
           return (out_vocab_file,)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__=None					)       ->   str:
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= [self.bos_token_id] if self.add_bos_token else []
           SCREAMING_SNAKE_CASE__			:				Dict       						= [self.eos_token_id] if self.add_eos_token else []
  
           SCREAMING_SNAKE_CASE__			:				str       						= bos_token_id + token_ids_a + eos_token_id
  
           if token_ids_a is not None:
              SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= output + bos_token_id + token_ids_a + eos_token_id
  
           return output
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None ,  a__ = False					)       ->   List[int]:
           if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=a__ ,  token_ids_a=a__ ,  already_has_special_tokens=a__					)
  
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [1] if self.add_bos_token else []
           SCREAMING_SNAKE_CASE__			:				List[str]       						= [1] if self.add_eos_token else []
  
           if token_ids_a is None:
              return bos_token_id + ([0] * len(a__					)) + eos_token_id
           return (
               bos_token_id
               + ([0] * len(a__					))
               + eos_token_id
               + bos_token_id
               + ([0] * len(a__					))
               + eos_token_id
           )
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				Any       						= [self.bos_token_id] if self.add_bos_token else []
           SCREAMING_SNAKE_CASE__			:				Dict       						= [self.eos_token_id] if self.add_eos_token else []
  
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= [0] * len(bos_token_id + token_ids_a + eos_token_id					)
  
           if token_ids_a is not None:
              output += [1] * len(bos_token_id + token_ids_a + eos_token_id					)
  
           return output
  
  
  
  
  
  
        def    A__(self ,  a__					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= list(conversation.iter_texts()					)
           if not all([is_user for is_user, msg in dialogue[::2]]					) or not all(
               [not is_user for is_user, msg in dialogue[1::2]]					):
              raise ValueError(
                  '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''					)
  
           SCREAMING_SNAKE_CASE__			:				List[int]       						= []
           if len(conversation.past_user_inputs					) > 0:
              if not conversation.past_user_inputs[0].startswith(a__					) or E_SYS not in conversation.past_user_inputs[0]:
                 SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= (
                     B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                 )
           elif not dialogue[0][1].startswith(a__					) or E_SYS not in dialogue[0][1]:
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])
  
           dialog_tokens += sum(
               [
                   [self.bos_token_id]
                   + self.encode(
                       f"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """ ,  add_special_tokens=a__					)
                   + [self.eos_token_id]
                   for prompt, answer in zip(dialogue[::2] ,  dialogue[1::2]					)
               ] ,  [] ,  )
           if not (dialogue[-1][0]):
              raise ValueError(f"""Last message must be from user, got {dialogue[-1]["role"]}"""					)
           dialog_tokens += [self.bos_token_id] + self.encode(
               f"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}""" ,  add_special_tokens=a__					)
           return dialog_tokens
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import normalizers
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_splinter import SplinterTokenizer
  
  
  lowercase__			=	logging.get_logger(__name__)
  
  lowercase__			=	{'''vocab_file''': '''vocab.txt'''}
  
  lowercase__			=	{
      '''vocab_file''': {
          '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
          '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
          '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
          '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
      }
  }
  
  lowercase__			=	{
      '''tau/splinter-base''': 5_1_2,
      '''tau/splinter-base-qass''': 5_1_2,
      '''tau/splinter-large''': 5_1_2,
      '''tau/splinter-large-qass''': 5_1_2,
  }
  
  lowercase__			=	{
      '''tau/splinter-base''': {'''do_lower_case''': False},
      '''tau/splinter-base-qass''': {'''do_lower_case''': False},
      '''tau/splinter-large''': {'''do_lower_case''': False},
      '''tau/splinter-large-qass''': {'''do_lower_case''': False},
  }
  
  
  
  
  
  
  class 						lowercase     (   UpperCAmelCase_):
        __lowercase 		=  VOCAB_FILES_NAMES
        __lowercase 		=  PRETRAINED_VOCAB_FILES_MAP
        __lowercase 		=  PRETRAINED_INIT_CONFIGURATION
        __lowercase 		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __lowercase 		=  SplinterTokenizer
  
  
  
  
  
  
        def __init__(self ,  a__=None ,  a__=None ,  a__=True ,  a__="[UNK]" ,  a__="[SEP]" ,  a__="[PAD]" ,  a__="[CLS]" ,  a__="[MASK]" ,  a__="[QUESTION]" ,  a__=True ,  a__=None ,  **a__ ,  )       ->   Any:
           super().__init__(
               a__ ,  tokenizer_file=a__ ,  do_lower_case=a__ ,  unk_token=a__ ,  sep_token=a__ ,  pad_token=a__ ,  cls_token=a__ ,  mask_token=a__ ,  tokenize_chinese_chars=a__ ,  strip_accents=a__ ,  additional_special_tokens=(question_token,) ,  **a__ ,  )
  
           SCREAMING_SNAKE_CASE__			:				Tuple       						= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
           if (
               pre_tok_state.get('''lowercase''' ,  a__					) != do_lower_case
               or pre_tok_state.get('''strip_accents''' ,  a__					) != strip_accents
           ):
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= getattr(a__ ,  pre_tok_state.pop('''type'''					)					)
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= do_lower_case
              SCREAMING_SNAKE_CASE__			:				Optional[int]       						= strip_accents
              SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= pre_tok_class(**a__					)
  
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= do_lower_case
  
  
  
  
  
  
        @property
        def    A__(self					)       ->   List[str]:
           return self.convert_tokens_to_ids(self.question_token					)
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  
           SCREAMING_SNAKE_CASE__			:				Dict       						= [self.cls_token_id]
           SCREAMING_SNAKE_CASE__			:				Tuple       						= [self.sep_token_id]
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= [self.question_token_id] + [self.convert_tokens_to_ids('''.'''					)]
           if self.padding_side == "right":
              # Input is question-then-context
              return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
           else:
              # Input is context-then-question
              return cls + token_ids_a + sep + token_ids_a + question_suffix + sep
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [self.sep_token_id]
           SCREAMING_SNAKE_CASE__			:				Any       						= [self.cls_token_id]
           SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [self.question_token_id] + [self.convert_tokens_to_ids('''.'''					)]
           if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
  
           if self.padding_side == "right":
              # Input is question-then-context
              return len(cls + token_ids_a + question_suffix + sep					) * [0] + len(token_ids_a + sep					) * [1]
           else:
              # Input is context-then-question
              return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + question_suffix + sep					) * [1]
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ = None					)       ->   Tuple[str]:
           SCREAMING_SNAKE_CASE__			:				int       						= self._tokenizer.model.save(a__ ,  name=a__					)
           return tuple(a__					)
  
  