code: 
  
  
  
  import os
  from shutil import copyfile
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  if TYPE_CHECKING:
  	from transformers.pipelines.conversational import Conversation
  
  __A       					=		logging.get_logger(__name__)
  
  __A       					=		{"""vocab_file""": """tokenizer.model"""}
  
  __A       					=		{
      """vocab_file""": {
          """hf-internal-testing/llama-tokenizer""": """https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model""",
      },
      """tokenizer_file""": {
          """hf-internal-testing/llama-tokenizer""": """https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json""",
      },
  }
  __A       					=		{
      """hf-internal-testing/llama-tokenizer""": 2048,
  }
  __A       					=		"""▁"""
  
  __A       					=		"""[INST]""", """[/INST]"""
  __A       					=		"""<<SYS>>\n""", """\n<</SYS>>\n\n"""
  
  # fmt: off
  __A       					=		"""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
  answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
  that your responses are socially unbiased and positive in nature.
  
  If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
  correct. If you don't know the answer to a question, please don't share false information."""
  
  
  
  
  
  
  class 	lowercase		(				UpperCamelCase__):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	a__   :      Dict	     =							VOCAB_FILES_NAMES
  	a__   :      Optional[Any]	     =							PRETRAINED_VOCAB_FILES_MAP
  	a__   :      Optional[Any]	     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	a__   :      Dict	     =							["""input_ids""", """attention_mask"""]
  
  
  
  
  	def __init__(			self			:		Union[str, Any]		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Dict="<unk>"		,						__UpperCAmelCase			:		int="<s>"		,						__UpperCAmelCase			:		Any="</s>"		,						__UpperCAmelCase			:		Dict=None		,						__UpperCAmelCase			:		str = None		,						__UpperCAmelCase			:		Union[str, Any]=True		,						__UpperCAmelCase			:		Any=False		,						__UpperCAmelCase			:		Tuple=False		,						__UpperCAmelCase			:		Any=True		,						**__UpperCAmelCase			:		Tuple		,						)    ->				Optional[Any]:
  							UpperCAmelCase_=     {} if sp_model_kwargs is None else sp_model_kwargs
  							UpperCAmelCase_=     AddedToken(lowerCAmelCase__		,						lstrip=lowerCAmelCase__		,						rstrip=lowerCAmelCase__  ) if isinstance(lowerCAmelCase__		,						lowerCAmelCase__  ) else bos_token
  							UpperCAmelCase_=     AddedToken(lowerCAmelCase__		,						lstrip=lowerCAmelCase__		,						rstrip=lowerCAmelCase__  ) if isinstance(lowerCAmelCase__		,						lowerCAmelCase__  ) else eos_token
  							UpperCAmelCase_=     AddedToken(lowerCAmelCase__		,						lstrip=lowerCAmelCase__		,						rstrip=lowerCAmelCase__  ) if isinstance(lowerCAmelCase__		,						lowerCAmelCase__  ) else unk_token
  							UpperCAmelCase_=     AddedToken(lowerCAmelCase__		,						lstrip=lowerCAmelCase__		,						rstrip=lowerCAmelCase__  ) if isinstance(lowerCAmelCase__		,						lowerCAmelCase__  ) else pad_token
  							super().__init__(
  							    bos_token=lowerCAmelCase__		,						eos_token=lowerCAmelCase__		,						unk_token=lowerCAmelCase__		,						pad_token=lowerCAmelCase__		,						add_bos_token=lowerCAmelCase__		,						add_eos_token=lowerCAmelCase__		,						sp_model_kwargs=self.sp_model_kwargs		,						clean_up_tokenization_spaces=lowerCAmelCase__		,						legacy=lowerCAmelCase__		,						**lowerCAmelCase__		,						)
  							if legacy:
  													logger.warning_once(
  													    F"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
  													    """ read the related pull request available at https://github.com/huggingface/transformers/pull/24565"""  )
  							UpperCAmelCase_=     legacy
  							UpperCAmelCase_=     vocab_file
  							UpperCAmelCase_=     add_bos_token
  							UpperCAmelCase_=     add_eos_token
  							UpperCAmelCase_=     spm.SentencePieceProcessor(**self.sp_model_kwargs  )
  							self.sp_model.Load(lowerCAmelCase__  )
  
  
  
  
  	def __getstate__(			self			:		int  )    ->				int:
  							UpperCAmelCase_=     self.__dict__.copy()
  							UpperCAmelCase_=     None
  							UpperCAmelCase_=     self.sp_model.serialized_model_proto()
  							return state
  
  
  
  
  	def __setstate__(			self			:		List[Any]		,						__UpperCAmelCase			:		str  )    ->				Union[str, Any]:
  							UpperCAmelCase_=     d
  							UpperCAmelCase_=     spm.SentencePieceProcessor(**self.sp_model_kwargs  )
  							self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )
  
  
  
  
  	@property
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Tuple  )    ->				str:
  							return self.sp_model.get_piece_size()
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				Any:
  							UpperCAmelCase_=     {self.convert_ids_to_tokens(lowerCAmelCase__  ): i for i in range(self.vocab_size  )}
  							vocab.update(self.added_tokens_encoder  )
  							return vocab
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str		,						__UpperCAmelCase			:		Union[str, Any]		,						**__UpperCAmelCase			:		Optional[int]  )    ->				List[str]:
  
  
  
  
  							if not self.legacy:
  													UpperCAmelCase_=     SPIECE_UNDERLINE + text.replace(lowerCAmelCase__		,						""" """  )
  							return super().tokenize(lowerCAmelCase__		,						**lowerCAmelCase__  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Tuple		,						__UpperCAmelCase			:		List[str]  )    ->				List[Any]:
  
  
  
  
  							if not self.legacy:
  													UpperCAmelCase_=     text.startswith(lowerCAmelCase__  )
  													if is_first:
  																			UpperCAmelCase_=     text[1:]
  
  							UpperCAmelCase_=     self.sp_model.encode(lowerCAmelCase__		,						out_type=lowerCAmelCase__  )
  
  							if not self.legacy and not is_first and not text.startswith(""" """  ) and tokens[0].startswith(lowerCAmelCase__  ):
  													UpperCAmelCase_=     ([tokens[0][1:]] if len(tokens[0]  ) > 1 else []) + tokens[1:]
  							return tokens
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		int		,						__UpperCAmelCase			:		List[Any]  )    ->				Optional[Any]:
  							return self.sp_model.piece_to_id(lowerCAmelCase__  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict		,						__UpperCAmelCase			:		int  )    ->				Optional[Any]:
  							UpperCAmelCase_=     self.sp_model.IdToPiece(lowerCAmelCase__  )
  							return token
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any		,						__UpperCAmelCase			:		int  )    ->				Union[str, Any]:
  							UpperCAmelCase_=     []
  							UpperCAmelCase_=     ""
  							UpperCAmelCase_=     False
  							for i, token in enumerate(lowerCAmelCase__  ):
  													# make sure that special tokens are not decoded using sentencepiece model
  													if token in self.all_special_tokens:
  																			if not prev_is_special and i != 0:
  																									out_string += " "
  																			out_string += self.sp_model.decode(lowerCAmelCase__  ) + token
  																			UpperCAmelCase_=     True
  																			UpperCAmelCase_=     []
  													else:
  																			current_sub_tokens.append(lowerCAmelCase__  )
  																			UpperCAmelCase_=     False
  							out_string += self.sp_model.decode(lowerCAmelCase__  )
  							return out_string
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		int		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		str = None  )    ->				Tuple[str]:
  
  
  
  
  							if not os.path.isdir(lowerCAmelCase__  ):
  													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
  													return
  							UpperCAmelCase_=     os.path.join(
  							    lowerCAmelCase__		,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
  
  							if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowerCAmelCase__  ) and os.path.isfile(self.vocab_file  ):
  													copyfile(self.vocab_file		,						lowerCAmelCase__  )
  							elif not os.path.isfile(self.vocab_file  ):
  													with open(lowerCAmelCase__		,						"""wb"""  ) as fi:
  																			UpperCAmelCase_=     self.sp_model.serialized_model_proto()
  																			fi.write(lowerCAmelCase__  )
  
  							return (out_vocab_file,)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		str=None  )    ->				str:
  							UpperCAmelCase_=     [self.bos_token_id] if self.add_bos_token else []
  							UpperCAmelCase_=     [self.eos_token_id] if self.add_eos_token else []
  
  							UpperCAmelCase_=     bos_token_id + token_ids_a + eos_token_id
  
  							if token_ids_a is not None:
  													UpperCAmelCase_=     output + bos_token_id + token_ids_a + eos_token_id
  
  							return output
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Union[str, Any] = None		,						__UpperCAmelCase			:		Union[str, Any] = False  )    ->				List[int]:
  
  
  
  
  							if already_has_special_tokens:
  													return super().get_special_tokens_mask(
  													    token_ids_a=lowerCAmelCase__		,						token_ids_a=lowerCAmelCase__		,						already_has_special_tokens=lowerCAmelCase__  )
  
  							UpperCAmelCase_=     [1] if self.add_bos_token else []
  							UpperCAmelCase_=     [1] if self.add_eos_token else []
  
  							if token_ids_a is None:
  													return bos_token_id + ([0] * len(lowerCAmelCase__  )) + eos_token_id
  							return (
  							    bos_token_id
  							    + ([0] * len(lowerCAmelCase__  ))
  							    + eos_token_id
  							    + bos_token_id
  							    + ([0] * len(lowerCAmelCase__  ))
  							    + eos_token_id
  							)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Dict = None  )    ->				List[int]:
  							UpperCAmelCase_=     [self.bos_token_id] if self.add_bos_token else []
  							UpperCAmelCase_=     [self.eos_token_id] if self.add_eos_token else []
  
  							UpperCAmelCase_=     [0] * len(bos_token_id + token_ids_a + eos_token_id  )
  
  							if token_ids_a is not None:
  													output += [1] * len(bos_token_id + token_ids_a + eos_token_id  )
  
  							return output
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict		,						__UpperCAmelCase			:		List[Any]  )    ->				List[int]:
  							UpperCAmelCase_=     list(conversation.iter_texts()  )
  							if not all([is_user for is_user, msg in dialogue[::2]]  ) or not all(
  							    [not is_user for is_user, msg in dialogue[1::2]]  ):
  													raise ValueError(
  													    """The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)"""  )
  
  							UpperCAmelCase_=     []
  							if len(conversation.past_user_inputs  ) > 0:
  													if not conversation.past_user_inputs[0].startswith(lowerCAmelCase__  ) or E_SYS not in conversation.past_user_inputs[0]:
  																			UpperCAmelCase_=     (
  																			    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
  																			)
  							elif not dialogue[0][1].startswith(lowerCAmelCase__  ) or E_SYS not in dialogue[0][1]:
  													UpperCAmelCase_=     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])
  
  							dialog_tokens += sum(
  							    [
  							        [self.bos_token_id]
  							        + self.encode(
  							            F"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """		,						add_special_tokens=lowerCAmelCase__  )
  							        + [self.eos_token_id]
  							        for prompt, answer in zip(dialogue[::2]		,						dialogue[1::2]  )
  							    ]		,						[]		,						)
  							if not (dialogue[-1][0]):
  													raise ValueError(F"""Last message must be from user, got {dialogue[-1]["role"]}"""  )
  							dialog_tokens += [self.bos_token_id] + self.encode(
  							    F"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""		,						add_special_tokens=lowerCAmelCase__  )
  							return dialog_tokens
  
  
  
  
  
style_context: 
  
  
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  __A       					=		logging.get_logger(__name__)
  
  __A       					=		{
      '''ZinengTang/tvlt-base''': '''https://huggingface.co/ZinengTang/tvlt-base/blob/main/config.json''',
  }
  
  
  
  
  
  
  class 	lowercase		(				snake_case__):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	a__   :      Tuple	     =							"tvlt"
  
  
  
  
  	def __init__(			self			:		List[str]		,						__UpperCAmelCase			:		List[str]=224		,						__UpperCAmelCase			:		List[str]=2_048		,						__UpperCAmelCase			:		Tuple=128		,						__UpperCAmelCase			:		Union[str, Any]=[16, 16]		,						__UpperCAmelCase			:		str=[16, 16]		,						__UpperCAmelCase			:		str=3		,						__UpperCAmelCase			:		List[str]=1		,						__UpperCAmelCase			:		Tuple=8		,						__UpperCAmelCase			:		Optional[Any]=768		,						__UpperCAmelCase			:		List[Any]=12		,						__UpperCAmelCase			:		Tuple=12		,						__UpperCAmelCase			:		List[str]=3_072		,						__UpperCAmelCase			:		List[Any]="gelu"		,						__UpperCAmelCase			:		str=0.0		,						__UpperCAmelCase			:		str=0.0		,						__UpperCAmelCase			:		List[Any]=0.02		,						__UpperCAmelCase			:		Optional[int]=1E-6		,						__UpperCAmelCase			:		Tuple=True		,						__UpperCAmelCase			:		List[Any]=False		,						__UpperCAmelCase			:		Union[str, Any]=16		,						__UpperCAmelCase			:		Any=512		,						__UpperCAmelCase			:		Optional[Any]=8		,						__UpperCAmelCase			:		Dict=2_048		,						__UpperCAmelCase			:		Dict=0.75		,						__UpperCAmelCase			:		Union[str, Any]=0.15		,						__UpperCAmelCase			:		Any="frame-level"		,						__UpperCAmelCase			:		int=True		,						__UpperCAmelCase			:		List[Any]=True		,						__UpperCAmelCase			:		Union[str, Any]="classification"		,						**__UpperCAmelCase			:		Any		,						)    ->				Tuple:
  							super().__init__(**__UpperCAmelCase  )
  
  							if audio_mask_type not in ("frame-level", "patch_level"):
  													raise ValueError(
  													    """audio_mask_type must be one of two acceptable strategies - {'frame_level', 'patch-level') """
  													    F"""got {audio_mask_type}"""  )
  
  							UpperCAmelCase_=     image_size
  							UpperCAmelCase_=     spectrogram_length
  							UpperCAmelCase_=     frequency_length
  							UpperCAmelCase_=     image_patch_size
  							UpperCAmelCase_=     audio_patch_size
  							UpperCAmelCase_=     num_image_channels
  							UpperCAmelCase_=     num_audio_channels
  							UpperCAmelCase_=     num_frames
  
  							UpperCAmelCase_=     hidden_size
  							UpperCAmelCase_=     num_hidden_layers
  							UpperCAmelCase_=     num_attention_heads
  							UpperCAmelCase_=     intermediate_size
  							UpperCAmelCase_=     hidden_act
  							UpperCAmelCase_=     hidden_dropout_prob
  							UpperCAmelCase_=     attention_probs_dropout_prob
  							UpperCAmelCase_=     initializer_range
  							UpperCAmelCase_=     layer_norm_eps
  							UpperCAmelCase_=     qkv_bias
  							UpperCAmelCase_=     use_mean_pooling
  
  							UpperCAmelCase_=     decoder_num_attention_heads
  							UpperCAmelCase_=     decoder_hidden_size
  							UpperCAmelCase_=     decoder_num_hidden_layers
  							UpperCAmelCase_=     decoder_intermediate_size
  							UpperCAmelCase_=     pixel_mask_ratio
  							UpperCAmelCase_=     audio_mask_ratio
  							UpperCAmelCase_=     audio_mask_type
  
  							UpperCAmelCase_=     task_matching
  							UpperCAmelCase_=     task_mae
  							UpperCAmelCase_=     loss_type
  
  
  
  
  