



"""simple docstring"""






def      snake_case__   (		__lowerCamelCase				:    bytes	):
			"""simple docstring"""
			return "".join([hex(__lowerCamelCase	)[2:].zfill(2	).upper() for byte in list(__lowerCamelCase	)]	)






def      snake_case__   (		__lowerCamelCase				:    str	):
			"""simple docstring"""
			# Check data validity, following RFC3548
			# https://www.ietf.org/rfc/rfc3548.txt
			if (len(__lowerCamelCase	) % 2) != 0:
						raise ValueError(
						    '''Base16 encoded data is invalid:
Data does not have an even number of hex digits.'''	)
			# Check the character set - the standard base16 alphabet
			# is uppercase according to RFC3548 section 6
			if not set(__lowerCamelCase	) <= set('''0123456789ABCDEF'''	):
						raise ValueError(
						    '''Base16 encoded data is invalid:
Data is not uppercase hex or it contains invalid characters.'''	)
			# For every two hexadecimal digits (= a byte), turn it into an integer.
			# Then, string the result together into bytes, and return it.
			return bytes(int(data[i] + data[i + 1]							,		16	) for i in range(0							,		len(__lowerCamelCase	)							,		2	)	)


if __name__ == "__main__":
		import doctest

		doctest.testmod()









"""simple docstring"""






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
		import torch


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


_lowercase : Optional[int]      						=						logging.get_logger(__name__)
_lowercase : Optional[int]      						=						{"vocab_file": "spiece.model"}

_lowercase : str      						=						{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_lowercase : Optional[Any]      						=						{
    "AI-Sweden/gpt-sw3-126m": 2_0_4_8,
    "AI-Sweden/gpt-sw3-350m": 2_0_4_8,
    "AI-Sweden/gpt-sw3-1.6b": 2_0_4_8,
    "AI-Sweden/gpt-sw3-6.7b": 2_0_4_8,
    "AI-Sweden/gpt-sw3-20b": 2_0_4_8,
}

class        __SCREAMING_SNAKE_CASE	(      lowerCAmelCase_   ):


					'''simple docstring'''





					_a   			=							VOCAB_FILES_NAMES
					_a   			=							PRETRAINED_VOCAB_FILES_MAP
					_a   			=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_a   			=							['input_ids', 'attention_mask']



					def __init__(       self							:  Dict,  lowerCamelCase							:  Tuple,  lowerCamelCase							:  List[str]=False,  lowerCamelCase							:  str=False,  lowerCamelCase							:  List[Any]=False,  lowerCamelCase							:  List[str]=None,  lowerCamelCase							:  Optional[Any]=None,  lowerCamelCase							:  List[str]=None,  lowerCamelCase							:  Optional[int]=None,  lowerCamelCase							:  Optional[Dict[str, Any]] = None,  **lowerCamelCase							:  Dict,  )->					None:
								lowerCamelCase__		:		Any			={} if sp_model_kwargs is None else sp_model_kwargs

								lowerCamelCase__		:		Optional[int]			=kwargs.get('''name_or_path'''     )
								if name_or_path is None:
											logger.warning(
											    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
											    ''' you are testing the model, this can safely be ignored'''     )
											lowerCamelCase__		:		List[Any]			='''None'''

								# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
								lowerCamelCase__		:		Tuple			='''<|endoftext|>''' if eos_token is None else eos_token
								lowerCamelCase__		:		Dict			='''<unk>''' if unk_token is None else unk_token
								if "gpt-sw3-7b" in name_or_path:
											lowerCamelCase__		:		int			=unk_token if pad_token is None else pad_token
											lowerCamelCase__		:		str			=eos_token if bos_token is None else bos_token
								else:
											lowerCamelCase__		:		Any			='''<pad>''' if pad_token is None else pad_token
											lowerCamelCase__		:		str			='''<s>''' if bos_token is None else bos_token

								super().__init__(
								    do_lower_case=lowerCamelCase,  remove_space=lowerCamelCase,  keep_accents=lowerCamelCase,  bos_token=lowerCamelCase,  eos_token=lowerCamelCase,  unk_token=lowerCamelCase,  pad_token=lowerCamelCase,  sp_model_kwargs=self.sp_model_kwargs,  **lowerCamelCase,  )

								lowerCamelCase__		:		Union[str, Any]			=do_lower_case
								lowerCamelCase__		:		Union[str, Any]			=remove_space
								lowerCamelCase__		:		int			=keep_accents
								lowerCamelCase__		:		Tuple			=vocab_file

								lowerCamelCase__		:		List[str]			=spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(lowerCamelCase     )

								# Used for whitespace normalization in input texts
								# fmt : off
								lowerCamelCase__		:		List[Any]			={''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
								# fmt : on

								# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
								lowerCamelCase__		:		int			=re.compile(
								    F'''[{"".join(map(lowerCamelCase,  list(range(0,  9     )     ) + list(range(11,  32     )     ) + list(range(127,  160     )     ) + [160, 173, 8203]     )     )}]'''     )



					def __getstate__(       self							:  Any     )->					Optional[Any]:
								lowerCamelCase__		:		List[str]			=self.__dict__.copy()
								lowerCamelCase__		:		List[Any]			=None
								return state



					def __setstate__(       self							:  Optional[Any],  lowerCamelCase							:  Dict     )->					int:
								lowerCamelCase__		:		List[str]			=d

								# for backward compatibility
								if not hasattr(self,  '''sp_model_kwargs'''     ):
											lowerCamelCase__		:		Union[str, Any]			={}

								lowerCamelCase__		:		str			=spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(self.vocab_file     )



					@property
					# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
					def 		snake_case       (       self							:  List[str]     )->					int:
								return len(self.sp_model     )



					def 		snake_case       (       self							:  List[str],  lowerCamelCase							:  str     )->					str:
								lowerCamelCase__		:		Dict			=self.non_printing_characters_re.sub('''''',  lowerCamelCase     )

								# Normalize whitespaces
								lowerCamelCase__		:		List[Any]			=''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]     )

								# NFC Unicode normalization
								lowerCamelCase__		:		Union[str, Any]			=unicodedata.normalize('''NFC''',  lowerCamelCase     )
								return text



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  str,  **lowerCamelCase							:  str     )->					List[str]:
								lowerCamelCase__		:		int			=self.preprocess_text(lowerCamelCase     )
								return self.sp_model.encode(lowerCamelCase,  out_type=lowerCamelCase     )



					def 		snake_case       (       self							:  Any,  lowerCamelCase							:  str     )->					int:
								return self.sp_model.PieceToId(lowerCamelCase     )



					def 		snake_case       (       self							:  Optional[int],  lowerCamelCase							:  int     )->					str:
								return self.sp_model.IdToPiece(lowerCamelCase     )



					@staticmethod
					def 		snake_case       (       lowerCamelCase							:  str     )->					str:
								return out_string



					def 		snake_case       (       self							:  List[Any],  lowerCamelCase							:  List[str]     )->					str:
								lowerCamelCase__		:		Dict			=[]
								lowerCamelCase__		:		int			=''''''
								lowerCamelCase__		:		Any			=False
								for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
														# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
														if not prev_is_special:
																	out_string += " "

														out_string += self.sp_model.decode(lowerCamelCase     ) + token
														lowerCamelCase__		:		Tuple			=True
														lowerCamelCase__		:		Tuple			=[]
											else:
														current_sub_tokens.append(lowerCamelCase     )
														lowerCamelCase__		:		Optional[Any]			=False
								out_string += self.sp_model.decode(lowerCamelCase     )

								return out_string



					def 		snake_case       (       self							:  Union[str, Any]     )->					Dict[str, int]:
								lowerCamelCase__		:		Tuple			={self.convert_ids_to_tokens(lowerCamelCase     ): i for i in range(self.vocab_size     )}
								vocab.update(self.added_tokens_encoder     )
								return vocab



					def 		snake_case       (       self							:  Optional[Any],  lowerCamelCase							:  str,  lowerCamelCase							:  Optional[str] = None     )->					Tuple[str]:
								if not os.path.isdir(lowerCamelCase     ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
											return
								lowerCamelCase__		:		List[str]			=os.path.join(
								    lowerCamelCase,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

								if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCamelCase     ) and os.path.isfile(self.vocab_file     ):
											copyfile(self.vocab_file,  lowerCamelCase     )
								elif not os.path.isfile(self.vocab_file     ):
											with open(lowerCamelCase,  '''wb'''     ) as fi:
														lowerCamelCase__		:		Dict			=self.sp_model.serialized_model_proto()
														fi.write(lowerCamelCase     )

								return (out_vocab_file,)



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  Union[str, List[str]],  lowerCamelCase							:  Union[str, bool] = False     )->					Union[List[int], List[List[int]], "torch.Tensor"]:

								if isinstance(lowerCamelCase,  lowerCamelCase     ):
											lowerCamelCase__		:		str			=self.preprocess_text(lowerCamelCase     )
											lowerCamelCase__		:		Union[str, Any]			=self.sp_model.encode(lowerCamelCase     )
								else:
											lowerCamelCase__		:		int			=[self.preprocess_text(lowerCamelCase     ) for t in text]
											lowerCamelCase__		:		Tuple			=self.sp_model.encode(lowerCamelCase     )

								if return_tensors is True or return_tensors == "pt":
											lowerCamelCase__		:		Union[str, Any]			=torch.tensor(lowerCamelCase     )

								return token_ids



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  Union[int, List[int]]     )->					str:
								return self.sp_model.decode(lowerCamelCase     )




					def 		snake_case       (       self							:  Tuple,  lowerCamelCase							:  "Conversation"     )->					List[int]:
								lowerCamelCase__		:		Any			=[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
								lowerCamelCase__		:		str			=(
								    F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(lowerCamelCase     ) + F'''{self.bos_token}Bot:'''
								)
								return self.encode(text=lowerCamelCase     )




