






"""simple docstring"""





import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

require_version("""tokenizers>=0.13.3""")

if is_sentencepiece_available():
  from .tokenization_llama import LlamaTokenizer
else:
  UpperCamelCase					=     None

UpperCamelCase					=     logging.get_logger(__name__)
UpperCamelCase					=     {"""vocab_file""": """tokenizer.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase,     UpperCamelCase					=     """[INST]""", """[/INST]"""
UpperCamelCase,     UpperCamelCase					=     """<<SYS>>\n""", """\n<</SYS>>\n\n"""

# fmt: off
UpperCamelCase					=     """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don't know the answer to a question, please don't share false information."""







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							VOCAB_FILES_NAMES
   UpperCamelCase__     	=							LlamaTokenizer
   UpperCamelCase__     	=							"""left"""
   UpperCamelCase__     	=							["""input_ids""", """attention_mask"""]





   def __init__(       self					, UpperCAmelCase_=None					, UpperCAmelCase_=None					, UpperCAmelCase_=False					, UpperCAmelCase_="<unk>"					, UpperCAmelCase_="<s>"					, UpperCAmelCase_="</s>"					, UpperCAmelCase_=True					, UpperCAmelCase_=False					, **UpperCAmelCase_					, )			->   Optional[Any]:




          '''simple docstring'''




          super().__init__(
              vocab_file=UpperCAmelCase_					, tokenizer_file=UpperCAmelCase_					, clean_up_tokenization_spaces=UpperCAmelCase_					, unk_token=UpperCAmelCase_					, bos_token=UpperCAmelCase_					, eos_token=UpperCAmelCase_					, **UpperCAmelCase_					, )
          lowercase__:							Union[str, Any]         =     add_bos_token
          lowercase__:							List[str]         =     add_eos_token
          self.update_post_processor()

          lowercase__:							Dict         =     vocab_file
          lowercase__:							str         =     False if not self.vocab_file else True





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          lowercase__:							Any         =     self.bos_token
          lowercase__:							str         =     self.bos_token_id

          lowercase__:							Optional[int]         =     self.eos_token
          lowercase__:							Any         =     self.eos_token_id

          lowercase__:							Any         =     F"""{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"""
          lowercase__:							Union[str, Any]         =     F"""{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"""

          lowercase__:							List[str]         =     []
          if self.add_bos_token:
                 special_tokens.append((bos, bos_token_id))
          if self.add_eos_token:
                 special_tokens.append((eos, eos_token_id))
          lowercase__:							List[str]         =     processors.TemplateProcessing(
              single=UpperCAmelCase_					, pair=UpperCAmelCase_					, special_tokens=UpperCAmelCase_)





   @property
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          return self._add_eos_token





   @property
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          return self._add_bos_token





   @add_eos_token.setter
   def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Union[str, Any]         =     value
          self.update_post_processor()





   @add_bos_token.setter
   def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							str         =     value
          self.update_post_processor()





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_ = None)			->   Tuple[str]:




          '''simple docstring'''




          if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                     "tokenizer.")

          if not os.path.isdir(UpperCAmelCase_):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
                 return
          lowercase__:							str         =     os.path.join(
              UpperCAmelCase_					, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

          if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCAmelCase_):
                 copyfile(self.vocab_file					, UpperCAmelCase_)

          return (out_vocab_file,)





   def __lowercase							(       self					, UpperCAmelCase_)			->   List[str]:




          '''simple docstring'''




          lowercase__:							int         =     list(conversation.iter_texts())
          if not all([is_user for is_user, msg in dialogue[::2]]) or not all(
              [not is_user for is_user, msg in dialogue[1::2]]):
                 raise ValueError(
                     "The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)")

          lowercase__:							str         =     []
          if len(conversation.past_user_inputs) > 0:
                 if not conversation.past_user_inputs[0].startswith(UpperCAmelCase_) or E_SYS not in conversation.past_user_inputs[0]:
                        lowercase__:							Optional[int]         =     (
                            B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                        )
          elif not dialogue[0][1].startswith(UpperCAmelCase_) or E_SYS not in dialogue[0][1]:
                 lowercase__:							Optional[Any]         =     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

          dialog_tokens += sum(
              [
                  [self.bos_token_id]
                  + self.encode(
                      F"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """					, add_special_tokens=UpperCAmelCase_)
                  + [self.eos_token_id]
                  for prompt, answer in zip(dialogue[::2]					, dialogue[1::2])
              ]					, []					, )
          if not (dialogue[-1][0]):
                 raise ValueError(F"""Last message must be from user, got {dialogue[-1]['role']}""")
          dialog_tokens += [self.bos_token_id] + self.encode(
              F"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""					, add_special_tokens=UpperCAmelCase_)
          return dialog_tokens














"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase					=     logging.get_logger(__name__)

UpperCamelCase					=     {
    """squeezebert/squeezebert-uncased""": (
        """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/config.json"""
    ),
    """squeezebert/squeezebert-mnli""": """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/config.json""",
    """squeezebert/squeezebert-mnli-headless""": (
        """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/config.json"""
    ),
}







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
   UpperCamelCase__     	=							"""squeezebert"""





   def __init__(       self					, UpperCAmelCase_=30_522					, UpperCAmelCase_=768					, UpperCAmelCase_=12					, UpperCAmelCase_=12					, UpperCAmelCase_=3_072					, UpperCAmelCase_="gelu"					, UpperCAmelCase_=0.1					, UpperCAmelCase_=0.1					, UpperCAmelCase_=512					, UpperCAmelCase_=2					, UpperCAmelCase_=0.02					, UpperCAmelCase_=1E-1_2					, UpperCAmelCase_=0					, UpperCAmelCase_=768					, UpperCAmelCase_=4					, UpperCAmelCase_=4					, UpperCAmelCase_=4					, UpperCAmelCase_=1					, UpperCAmelCase_=4					, UpperCAmelCase_=4					, **UpperCAmelCase_					, )			->   int:




          '''simple docstring'''




          super().__init__(pad_token_id=UpperCAmelCase_					, **UpperCAmelCase_)

          lowercase__:							Union[str, Any]         =     vocab_size
          lowercase__:							int         =     hidden_size
          lowercase__:							int         =     num_hidden_layers
          lowercase__:							List[str]         =     num_attention_heads
          lowercase__:							Any         =     hidden_act
          lowercase__:							List[str]         =     intermediate_size
          lowercase__:							Any         =     hidden_dropout_prob
          lowercase__:							List[Any]         =     attention_probs_dropout_prob
          lowercase__:							Optional[int]         =     max_position_embeddings
          lowercase__:							List[Any]         =     type_vocab_size
          lowercase__:							Union[str, Any]         =     initializer_range
          lowercase__:							str         =     layer_norm_eps
          lowercase__:							Optional[Any]         =     embedding_size
          lowercase__:							Optional[int]         =     q_groups
          lowercase__:							List[str]         =     k_groups
          lowercase__:							Union[str, Any]         =     v_groups
          lowercase__:							str         =     post_attention_groups
          lowercase__:							Optional[int]         =     intermediate_groups
          lowercase__:							List[Any]         =     output_groups







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   @property
   def __lowercase							(       self)			->   Mapping[str, Mapping[int, str]]:




          '''simple docstring'''




          if self.task == "multiple-choice":
                 lowercase__:							Optional[int]         =     {0: "batch", 1: "choice", 2: "sequence"}
          else:
                 lowercase__:							Optional[Any]         =     {0: "batch", 1: "sequence"}
          return OrderedDict(
              [
                  ("input_ids", dynamic_axis),
                  ("attention_mask", dynamic_axis),
                  ("token_type_ids", dynamic_axis),
              ])






