





"""simple docstring"""





import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
  import importlib.resources as pkg_resources
except ImportError:
  # Try backported to PY<37 `importlib_resources`.
  import importlib_resources as pkg_resources

from . import resources


SCREAMING_SNAKE_CASE				:     Optional[Any]          =	'''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
SCREAMING_SNAKE_CASE				:     List[str]          =	F'{BASE_REF_URL}/{__file__}'
SCREAMING_SNAKE_CASE				:     int          =	logging.getLogger(__name__)






def 	__UpperCAmelCase					(       snake_case_  :  str   )							->	Tuple[Any, str]:






   """simple docstring"""


   _lowerCAmelCase					     =		pkg_resources.read_text(snake_case_   ,	snake_case_   )
   return yaml.safe_load(snake_case_   ), F"""{BASE_REF_URL}/resources/{resource}"""


SCREAMING_SNAKE_CASE				:     str          =	load_yaml_resource('''readme_structure.yaml''')

SCREAMING_SNAKE_CASE				:     Optional[int]          =	[
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
SCREAMING_SNAKE_CASE				:     Any          =	Tuple[dict, List[str], List[str]]


class       __lowerCamelCase	:







   def __init__(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase = None       ,	lowerCamelCase = False				):




      '''simple docstring'''




      _lowerCAmelCase					     =		name
      _lowerCAmelCase					     =		level
      _lowerCAmelCase					     =		lines
      _lowerCAmelCase					     =		""""""
      _lowerCAmelCase					     =		True
      _lowerCAmelCase					     =		{}
      _lowerCAmelCase					     =		[]
      _lowerCAmelCase					     =		[]
      if self.lines is not None:
         self.parse(suppress_parsing_errors=lowerCamelCase				)







   def 							A__						(self       ,	lowerCamelCase = False				):




      '''simple docstring'''




      _lowerCAmelCase					     =		""""""
      _lowerCAmelCase					     =		[]
      _lowerCAmelCase					     =		False
      for line in self.lines:
         if line.strip(""" \n"""				) == "":
            continue
         elif line.strip(""" \n"""				)[:3] == "```":
            _lowerCAmelCase					     =		not code_start
         elif line.split()[0] == self.level + "#" and not code_start:
            if current_sub_level != "":
               _lowerCAmelCase					     =		Section(lowerCamelCase       ,	self.level + """#"""       ,	lowerCamelCase				)
               _lowerCAmelCase					     =		[]
            else:
               if current_lines != []:
                  self.text += "".join(lowerCamelCase				).strip()
                  if self.text != "" and self.text not in FILLER_TEXT:
                     _lowerCAmelCase					     =		False
                  _lowerCAmelCase					     =		[]

            _lowerCAmelCase					     =		""" """.join(line.split()[1:]				).strip(""" \n"""				)
         else:
            current_lines.append(lowerCamelCase				)
      else:
         if current_sub_level != "":
            if current_sub_level in self.content:
               self.parsing_error_list.append(
                   f"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""				)
            _lowerCAmelCase					     =		Section(lowerCamelCase       ,	self.level + """#"""       ,	lowerCamelCase				)
         else:
            if current_lines != []:
               self.text += "".join(lowerCamelCase				).strip()
               if self.text != "" and self.text not in FILLER_TEXT:
                  _lowerCAmelCase					     =		False

      if self.level == "" and not suppress_parsing_errors:
         if self.parsing_error_list != [] or self.parsing_warning_list != []:
            _lowerCAmelCase					     =		_lowerCAmelCase					     =		"""\n""".join("""-\t""" + x for x in self.parsing_error_list + self.parsing_warning_list				)
            _lowerCAmelCase					     =		f"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
            raise ValueError(lowerCamelCase				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[]
      _lowerCAmelCase					     =		[]
      if structure["allow_empty"] is False:
         # If content is expected
         if self.is_empty_text and self.content == {}:
            # If no content is found, mention it in the error_list
            error_list.append(f"""Expected some content in section `{self.name}` but it is empty."""				)

      if structure["allow_empty_text"] is False:
         # If some text is expected
         if self.is_empty_text:
            # If no text is found, mention it in the error_list
            error_list.append(
                f"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""				)
        # Subsections Validation
      if structure["subsections"] is not None:
         # If subsections are expected
         if self.content == {}:
            # If no subsections are present
            _lowerCAmelCase					     =		[subsection["""name"""] for subsection in structure["""subsections"""]]
            # Mention the expected values in the error_list
            error_list.append(
                f"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]				)}. Found 'None'."""				)
         else:
            # If some subsections are present
            _lowerCAmelCase					     =		[subsection["""name"""] for subsection in structure["""subsections"""]]
            _lowerCAmelCase					     =		False
            for idx, name in enumerate(lowerCamelCase				):
               if name not in self.content:
                  # If the expected subsection is not present
                  error_list.append(f"""Section `{self.name}` is missing subsection: `{name}`."""				)
                  _lowerCAmelCase					     =		True
               else:
                  # If the subsection is present, validate subsection, return the result
                  # and concat the errors from subsection to section error_list

                  # Skip sublevel validation if current level is `###`
                  if self.level == "###":
                     continue
                  else:
                     _lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase					     =		self.content[name].validate(
                         structure["""subsections"""][idx]				)
                  error_list += subsec_error_list
                  warning_list += subsec_warning_list

            if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
               for name in self.content:
                  if name not in structure_names:
                     # If an extra subsection is present
                     warning_list.append(
                         f"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""				)
      if error_list:
         # If there are errors, do not return the dictionary as it is invalid
         return {}, error_list, warning_list
      else:
         return self.to_dict(), error_list, warning_list







   def 							A__						(self				):




      '''simple docstring'''




      return {
          "name": self.name,
          "text": self.text,
          "is_empty_text": self.is_empty_text,
          "subsections": [value.to_dict() for value in self.content.values()],
      }





class       __lowerCamelCase	(   __lowercase    ):  # Level 0







   def __init__(self       ,	lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase = None       ,	lowerCamelCase = False				):




      '''simple docstring'''




      super().__init__(name=lowerCamelCase       ,	level=""""""				)  # Not using lines here as we need to use a child class parse
      _lowerCAmelCase					     =		structure
      _lowerCAmelCase					     =		-2
      _lowerCAmelCase					     =		0
      _lowerCAmelCase					     =		lines
      if self.lines is not None:
         self.parse(suppress_parsing_errors=lowerCamelCase				)







   def 							A__						(self				):




      '''simple docstring'''




      if self.structure is None:
         _lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase					     =		self._validate(lowerCamelCase				)
      else:
         _lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase					     =		self._validate(self.structure				)
      if error_list != [] or warning_list != []:
         _lowerCAmelCase					     =		"""\n""".join(["""-\t""" + x for x in error_list + warning_list]				)
         _lowerCAmelCase					     =		f"""The following issues were found for the README at `{self.name}`:\n""" + errors
         raise ValueError(lowerCamelCase				)







   @classmethod
   def 							A__						(cls       ,	lowerCamelCase       ,	lowerCamelCase = None       ,	lowerCamelCase = False				):




      '''simple docstring'''




      with open(lowerCamelCase       ,	encoding="""utf-8"""				) as f:
         _lowerCAmelCase					     =		f.readlines()
      return cls(lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	suppress_parsing_errors=lowerCamelCase				)







   @classmethod
   def 							A__						(cls       ,	lowerCamelCase       ,	lowerCamelCase = None       ,	lowerCamelCase = "root"       ,	lowerCamelCase = False				):




      '''simple docstring'''




      _lowerCAmelCase					     =		string.split("""\n"""				)
      return cls(lowerCamelCase       ,	lowerCamelCase       ,	lowerCamelCase       ,	suppress_parsing_errors=lowerCamelCase				)







   def 							A__						(self       ,	lowerCamelCase = False				):




      '''simple docstring'''




      _lowerCAmelCase					     =		0

      for line in self.lines:
         self.yaml_tags_line_count += 1
         if line.strip(""" \n"""				) == "---":
            self.tag_count += 1
            if self.tag_count == 2:
               break
         line_count += 1
      if self.tag_count == 2:
         _lowerCAmelCase					     =		self.lines[line_count + 1 :]  # Get the last + 1 th item.
      else:
         _lowerCAmelCase					     =		self.lines[self.tag_count :]
      super().parse(suppress_parsing_errors=lowerCamelCase				)







   def __str__(self				):




      '''simple docstring'''




      return str(self.to_dict()				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[]
      _lowerCAmelCase					     =		[]
      if self.yaml_tags_line_count == 0:
         warning_list.append("""Empty YAML markers are present in the README."""				)
      elif self.tag_count == 0:
         warning_list.append("""No YAML markers are present in the README."""				)
      elif self.tag_count == 1:
         warning_list.append("""Only the start of YAML tags present in the README."""				)
      # Check how many first level sections are present.
      _lowerCAmelCase					     =		len(self.content.keys()				)
      if num_first_level_keys > 1:
         # If more than one, add to the error list, continue
         error_list.append(
             f"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()				)]				)}. Only one heading is expected. Skipping further validation for this README."""				)
      elif num_first_level_keys < 1:
         # If less than one, append error.
         error_list.append(
             """The README has no first-level headings. One heading is expected. Skipping further validation for this README."""				)

      else:
         # If one exactly
         _lowerCAmelCase					     =		list(self.content.keys()				)[0]  # Get the key
         if start_key.startswith("""Dataset Card for"""				):  # Check correct start
            # If the starting is correct, validate all the sections
            _lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase					     =		self.content[start_key].validate(
                readme_structure["""subsections"""][0]				)
            error_list += sec_error_list
            warning_list += sec_warning_list
         else:
            # If not found, append error
            error_list.append(
                """No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."""				)
      if error_list:
         # If there are errors, do not return the dictionary as it is invalid
         return {}, error_list, warning_list
      else:
         return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
  from argparse import ArgumentParser

  SCREAMING_SNAKE_CASE				:     str          =	ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
  ap.add_argument('''readme_filepath''')
  SCREAMING_SNAKE_CASE				:     List[str]          =	ap.parse_args()
  SCREAMING_SNAKE_CASE				:     int          =	Path(args.readme_filepath)
  SCREAMING_SNAKE_CASE				:     Optional[Any]          =	ReadMe.from_readme(readme_filepath)




"""simple docstring"""





import os
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE				:     Optional[int]          =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE				:     List[str]          =	{'''vocab_file''': '''tokenizer.model'''}

SCREAMING_SNAKE_CASE				:     str          =	{
    '''vocab_file''': {
        '''hf-internal-testing/llama-tokenizer''': '''https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model''',
    },
    '''tokenizer_file''': {
        '''hf-internal-testing/llama-tokenizer''': '''https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json''',
    },
}
SCREAMING_SNAKE_CASE				:     Tuple          =	{
    '''hf-internal-testing/llama-tokenizer''': 2_0_4_8,
}
SCREAMING_SNAKE_CASE				:     Tuple          =	'''▁'''

SCREAMING_SNAKE_CASE    ,					SCREAMING_SNAKE_CASE				:     Optional[int]          =	'''[INST]''', '''[/INST]'''
SCREAMING_SNAKE_CASE    ,					SCREAMING_SNAKE_CASE				:     Union[str, Any]          =	'''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
SCREAMING_SNAKE_CASE				:     Optional[Any]          =	'''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information.'''


class       __lowerCamelCase	(   __lowercase    ):
   __UpperCamelCase									=   VOCAB_FILES_NAMES
   __UpperCamelCase									=   PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase									=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase									=   ['input_ids', 'attention_mask']







   def __init__(self       ,	lowerCamelCase       ,	lowerCamelCase="<unk>"       ,	lowerCamelCase="<s>"       ,	lowerCamelCase="</s>"       ,	lowerCamelCase=None       ,	lowerCamelCase = None       ,	lowerCamelCase=True       ,	lowerCamelCase=False       ,	lowerCamelCase=False       ,	lowerCamelCase=True       ,	**lowerCamelCase       ,	):




      '''simple docstring'''




      _lowerCAmelCase					     =		{} if sp_model_kwargs is None else sp_model_kwargs
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else bos_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else eos_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else unk_token
      _lowerCAmelCase					     =		AddedToken(lowerCamelCase       ,	lstrip=lowerCamelCase       ,	rstrip=lowerCamelCase				) if isinstance(lowerCamelCase       ,	lowerCamelCase				) else pad_token
      super().__init__(
          bos_token=lowerCamelCase       ,	eos_token=lowerCamelCase       ,	unk_token=lowerCamelCase       ,	pad_token=lowerCamelCase       ,	add_bos_token=lowerCamelCase       ,	add_eos_token=lowerCamelCase       ,	sp_model_kwargs=self.sp_model_kwargs       ,	clean_up_tokenization_spaces=lowerCamelCase       ,	legacy=lowerCamelCase       ,	**lowerCamelCase       ,	)
      if legacy:
         logger.warning_once(
             f"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
             """ read the related pull request available at https://github.com/huggingface/transformers/pull/24565"""				)
      _lowerCAmelCase					     =		legacy
      _lowerCAmelCase					     =		vocab_file
      _lowerCAmelCase					     =		add_bos_token
      _lowerCAmelCase					     =		add_eos_token
      _lowerCAmelCase					     =		spm.SentencePieceProcessor(**self.sp_model_kwargs				)
      self.sp_model.Load(lowerCamelCase				)







   def __getstate__(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.__dict__.copy()
      _lowerCAmelCase					     =		None
      _lowerCAmelCase					     =		self.sp_model.serialized_model_proto()
      return state







   def __setstate__(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		d
      _lowerCAmelCase					     =		spm.SentencePieceProcessor(**self.sp_model_kwargs				)
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto				)







   @property
   def 							A__						(self				):




      '''simple docstring'''




      return self.sp_model.get_piece_size()







   def 							A__						(self				):




      '''simple docstring'''




      _lowerCAmelCase					     =		{self.convert_ids_to_tokens(lowerCamelCase				): i for i in range(self.vocab_size				)}
      vocab.update(self.added_tokens_encoder				)
      return vocab







   def 							A__						(self       ,	lowerCamelCase       ,	**lowerCamelCase				):




      '''simple docstring'''




      if not self.legacy:
         _lowerCAmelCase					     =		SPIECE_UNDERLINE + text.replace(lowerCamelCase       ,	""" """				)
      return super().tokenize(lowerCamelCase       ,	**lowerCamelCase				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      if not self.legacy:
         _lowerCAmelCase					     =		text.startswith(lowerCamelCase				)
         if is_first:
            _lowerCAmelCase					     =		text[1:]

      _lowerCAmelCase					     =		self.sp_model.encode(lowerCamelCase       ,	out_type=lowerCamelCase				)

      if not self.legacy and not is_first and not text.startswith(""" """				) and tokens[0].startswith(lowerCamelCase				):
         _lowerCAmelCase					     =		([tokens[0][1:]] if len(tokens[0]				) > 1 else []) + tokens[1:]
      return tokens







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      return self.sp_model.piece_to_id(lowerCamelCase				)







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		self.sp_model.IdToPiece(lowerCamelCase				)
      return token







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[]
      _lowerCAmelCase					     =		""""""
      _lowerCAmelCase					     =		False
      for i, token in enumerate(lowerCamelCase				):
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            if not prev_is_special and i != 0:
               out_string += " "
            out_string += self.sp_model.decode(lowerCamelCase				) + token
            _lowerCAmelCase					     =		True
            _lowerCAmelCase					     =		[]
         else:
            current_sub_tokens.append(lowerCamelCase				)
            _lowerCAmelCase					     =		False
      out_string += self.sp_model.decode(lowerCamelCase				)
      return out_string







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None				):




      '''simple docstring'''




      if not os.path.isdir(lowerCamelCase				):
         logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
         return
      _lowerCAmelCase					     =		os.path.join(
          lowerCamelCase       ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

      if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCamelCase				) and os.path.isfile(self.vocab_file				):
         copyfile(self.vocab_file       ,	lowerCamelCase				)
      elif not os.path.isfile(self.vocab_file				):
         with open(lowerCamelCase       ,	"""wb"""				) as fi:
            _lowerCAmelCase					     =		self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase				)

      return (out_vocab_file,)







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase=None				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[self.bos_token_id] if self.add_bos_token else []
      _lowerCAmelCase					     =		[self.eos_token_id] if self.add_eos_token else []

      _lowerCAmelCase					     =		bos_token_id + token_ids_a + eos_token_id

      if token_ids_a is not None:
         _lowerCAmelCase					     =		output + bos_token_id + token_ids_a + eos_token_id

      return output







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None       ,	lowerCamelCase = False				):




      '''simple docstring'''




      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase       ,	token_ids_a=lowerCamelCase       ,	already_has_special_tokens=lowerCamelCase				)

      _lowerCAmelCase					     =		[1] if self.add_bos_token else []
      _lowerCAmelCase					     =		[1] if self.add_eos_token else []

      if token_ids_a is None:
         return bos_token_id + ([0] * len(lowerCamelCase				)) + eos_token_id
      return (
          bos_token_id
          + ([0] * len(lowerCamelCase				))
          + eos_token_id
          + bos_token_id
          + ([0] * len(lowerCamelCase				))
          + eos_token_id
      )







   def 							A__						(self       ,	lowerCamelCase       ,	lowerCamelCase = None				):




      '''simple docstring'''




      _lowerCAmelCase					     =		[self.bos_token_id] if self.add_bos_token else []
      _lowerCAmelCase					     =		[self.eos_token_id] if self.add_eos_token else []

      _lowerCAmelCase					     =		[0] * len(bos_token_id + token_ids_a + eos_token_id				)

      if token_ids_a is not None:
         output += [1] * len(bos_token_id + token_ids_a + eos_token_id				)

      return output







   def 							A__						(self       ,	lowerCamelCase				):




      '''simple docstring'''




      _lowerCAmelCase					     =		list(conversation.iter_texts()				)
      if not all([is_user for is_user, msg in dialogue[::2]]				) or not all(
          [not is_user for is_user, msg in dialogue[1::2]]				):
         raise ValueError(
             """The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)"""				)

      _lowerCAmelCase					     =		[]
      if len(conversation.past_user_inputs				) > 0:
         if not conversation.past_user_inputs[0].startswith(lowerCamelCase				) or E_SYS not in conversation.past_user_inputs[0]:
            _lowerCAmelCase					     =		(
                B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
            )
      elif not dialogue[0][1].startswith(lowerCamelCase				) or E_SYS not in dialogue[0][1]:
         _lowerCAmelCase					     =		(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

      dialog_tokens += sum(
          [
              [self.bos_token_id]
              + self.encode(
                  f"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """       ,	add_special_tokens=lowerCamelCase				)
              + [self.eos_token_id]
              for prompt, answer in zip(dialogue[::2]       ,	dialogue[1::2]				)
          ]       ,	[]       ,	)
      if not (dialogue[-1][0]):
         raise ValueError(f"""Last message must be from user, got {dialogue[-1]["role"]}"""				)
      dialog_tokens += [self.bos_token_id] + self.encode(
          f"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""       ,	add_special_tokens=lowerCamelCase				)
      return dialog_tokens