





import dataclasses
import json
import warnings
from dataclasses import dataclass, field
from time import time
from typing import List

from ..utils import logging


_a 						=      logging.get_logger(__name__)





def lowerCAmelCase__(__snake_case=None       ,__snake_case=None							)				->      Optional[Any]:







	'''simple docstring'''







	return field(default_factory=lambda: default       ,metadata=__snake_case							)






@dataclass
class 					__A    :

			'''simple docstring'''







			lowerCAmelCase_							     =						list_field(
			    default=[]			,				metadata={
			        """help""": (
			            """Model checkpoints to be provided to the AutoModel classes. Leave blank to benchmark the base version"""
			            """ of all available models"""
			        )
			    }			,				)

			lowerCAmelCase_							     =						list_field(
			    default=[8]			,				metadata={"""help""": """List of batch sizes for which memory and time performance will be evaluated"""} )

			lowerCAmelCase_							     =						list_field(
			    default=[8, 32, 128, 512]			,				metadata={"""help""": """List of sequence lengths for which memory and time performance will be evaluated"""}			,				)

			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={"""help""": """Whether to benchmark inference of model. Inference can be disabled via --no-inference."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={"""help""": """Whether to run on available cuda devices. Cuda can be disabled via --no-cuda."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={"""help""": """Whether to run on available tpu devices. TPU can be disabled via --no-tpu."""} )
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Use FP16 to accelerate inference."""} )
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Benchmark training of model"""} )
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Verbose memory tracing"""} )
			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={"""help""": """Whether to perform speed measurements. Speed measurements can be disabled via --no-speed."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={
			        """help""": """Whether to perform memory measurements. Memory measurements can be disabled via --no-memory"""
			    }			,				)
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Trace memory line by line"""} )
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Save result to a CSV file"""} )
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Save all print statements in a log file"""} )
			lowerCAmelCase_							     =						field(default=lowerCAmelCase			,				metadata={"""help""": """Whether to print environment information"""} )
			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={
			        """help""": (
			            """Whether to use multiprocessing for memory and speed measurement. It is highly recommended to use"""
			            """ multiprocessing for accurate CPU and GPU memory measurements. This option should only be disabled"""
			            """ for debugging / testing and on TPU."""
			        )
			    }			,				)
			lowerCAmelCase_							     =						field(
			    default=F"""inference_time_{round(time() )}.csv"""			,				metadata={"""help""": """CSV filename used if saving time results to csv."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=F"""inference_memory_{round(time() )}.csv"""			,				metadata={"""help""": """CSV filename used if saving memory results to csv."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=F"""train_time_{round(time() )}.csv"""			,				metadata={"""help""": """CSV filename used if saving time results to csv for training."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=F"""train_memory_{round(time() )}.csv"""			,				metadata={"""help""": """CSV filename used if saving memory results to csv for training."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=F"""env_info_{round(time() )}.csv"""			,				metadata={"""help""": """CSV filename used if saving environment information."""}			,				)
			lowerCAmelCase_							     =						field(
			    default=F"""log_{round(time() )}.csv"""			,				metadata={"""help""": """Log filename used if print statements are saved in log."""}			,				)
			lowerCAmelCase_							     =						field(default=3			,				metadata={"""help""": """Times an experiment will be run."""} )
			lowerCAmelCase_							     =						field(
			    default=lowerCAmelCase			,				metadata={
			        """help""": (
			            """Instead of loading the model as defined in `config.architectures` if exists, just load the pretrain"""
			            """ model weights."""
			        )
			    }			,				)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				warnings.warn(
				    F'The class {self.__class__} is deprecated. Hugging Face Benchmarking utils'
				    ''' are deprecated in general and it is advised to use external Benchmarking libraries '''
				    ''' to benchmark Transformer models.'''       ,  __lowerCAmelCase       ,  )





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return json.dumps(dataclasses.asdict(self							)       ,  indent=2							)





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				if len(self.models							) <= 0:
					raise ValueError(
					    '''Please make sure you provide at least one model name / model identifier, *e.g.* `--models'''
					    ''' bert-base-cased` or `args.models = [\'bert-base-cased\'].'''							)
				return self.models





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				if not self.multi_process:
					return False
				elif self.is_tpu:
					logger.info('''Multiprocessing is currently not possible on TPU.'''							)
					return False
				else:
					return True










import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
			import torch


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


_a 						=      logging.get_logger(__name__)
_a 						=      {"vocab_file": "spiece.model"}

_a 						=      {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_a 						=      {
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						VOCAB_FILES_NAMES
			lowerCAmelCase_							     =						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							     =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							     =						["""input_ids""", """attention_mask"""]





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase = None       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		{} if sp_model_kwargs is None else sp_model_kwargs

				lowerCamelCase__								=		kwargs.get('''name_or_path'''							)
				if name_or_path is None:
					logger.warning(
					    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
					    ''' you are testing the model, this can safely be ignored'''							)
					lowerCamelCase__								=		'''None'''

				# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
				lowerCamelCase__								=		'''<|endoftext|>''' if eos_token is None else eos_token
				lowerCamelCase__								=		'''<unk>''' if unk_token is None else unk_token
				if "gpt-sw3-7b" in name_or_path:
					lowerCamelCase__								=		unk_token if pad_token is None else pad_token
					lowerCamelCase__								=		eos_token if bos_token is None else bos_token
				else:
					lowerCamelCase__								=		'''<pad>''' if pad_token is None else pad_token
					lowerCamelCase__								=		'''<s>''' if bos_token is None else bos_token

				super().__init__(
				    do_lower_case=__lowerCAmelCase       ,  remove_space=__lowerCAmelCase       ,  keep_accents=__lowerCAmelCase       ,  bos_token=__lowerCAmelCase       ,  eos_token=__lowerCAmelCase       ,  unk_token=__lowerCAmelCase       ,  pad_token=__lowerCAmelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  **__lowerCAmelCase       ,  )

				lowerCamelCase__								=		do_lower_case
				lowerCamelCase__								=		remove_space
				lowerCamelCase__								=		keep_accents
				lowerCamelCase__								=		vocab_file

				lowerCamelCase__								=		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(__lowerCAmelCase							)

				# Used for whitespace normalization in input texts
				# fmt : off
				lowerCamelCase__								=		{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
				# fmt : on

				# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
				lowerCamelCase__								=		re.compile(
				    F'[{"".join(map(__lowerCAmelCase       ,  list(range(0       ,  9							)							) + list(range(1_1       ,  3_2							)							) + list(range(1_2_7       ,  1_6_0							)							) + [1_6_0, 1_7_3, 8_2_0_3]							)							)}]'							)





			def __getstate__(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.__dict__.copy()
				lowerCamelCase__								=		None
				return state





			def __setstate__(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		d

				# for backward compatibility
				if not hasattr(self       ,  '''sp_model_kwargs'''							):
					lowerCamelCase__								=		{}

				lowerCamelCase__								=		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(self.vocab_file							)





			@property
			# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return len(self.sp_model							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.non_printing_characters_re.sub(''''''       ,  __lowerCAmelCase							)

				# Normalize whitespaces
				lowerCamelCase__								=		''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]							)

				# NFC Unicode normalization
				lowerCamelCase__								=		unicodedata.normalize('''NFC'''       ,  __lowerCAmelCase							)
				return text





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.preprocess_text(__lowerCAmelCase							)
				return self.sp_model.encode(__lowerCAmelCase       ,  out_type=__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.PieceToId(__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.IdToPiece(__lowerCAmelCase							)





			@staticmethod
			def 		__lowerCamelCase				(    __lowerCAmelCase							):



				'''simple docstring'''







				return out_string





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[]
				lowerCamelCase__								=		''''''
				lowerCamelCase__								=		False
				for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
						if not prev_is_special:
							out_string += " "

						out_string += self.sp_model.decode(__lowerCAmelCase							) + token
						lowerCamelCase__								=		True
						lowerCamelCase__								=		[]
					else:
						current_sub_tokens.append(__lowerCAmelCase							)
						lowerCamelCase__								=		False
				out_string += self.sp_model.decode(__lowerCAmelCase							)

				return out_string





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		{self.convert_ids_to_tokens(__lowerCAmelCase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				if not os.path.isdir(__lowerCAmelCase							):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
					return
				lowerCamelCase__								=		os.path.join(
				    __lowerCAmelCase       ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

				if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCAmelCase							) and os.path.isfile(self.vocab_file							):
					copyfile(self.vocab_file       ,  __lowerCAmelCase							)
				elif not os.path.isfile(self.vocab_file							):
					with open(__lowerCAmelCase       ,  '''wb'''							) as fi:
						lowerCamelCase__								=		self.sp_model.serialized_model_proto()
						fi.write(__lowerCAmelCase							)

				return (out_vocab_file,)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = False							):



				'''simple docstring'''







				if isinstance(__lowerCAmelCase       ,  __lowerCAmelCase							):
					lowerCamelCase__								=		self.preprocess_text(__lowerCAmelCase							)
					lowerCamelCase__								=		self.sp_model.encode(__lowerCAmelCase							)
				else:
					lowerCamelCase__								=		[self.preprocess_text(__lowerCAmelCase							) for t in text]
					lowerCamelCase__								=		self.sp_model.encode(__lowerCAmelCase							)

				if return_tensors is True or return_tensors == "pt":
					lowerCamelCase__								=		torch.tensor(__lowerCAmelCase							)

				return token_ids





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.decode(__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
				lowerCamelCase__								=		(
				    F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(__lowerCAmelCase							) + F'{self.bos_token}Bot:'
				)
				return self.encode(text=__lowerCAmelCase							)



