





import datasets

from .evaluate import evaluate


_a 						=      "\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n"

_a 						=      "\nThis metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n"

_a 						=      "\nComputes SQuAD scores (F1 and EM).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair as given in the references (see below)\n        - 'prediction_text': the text of the answer\n    references: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair (see above),\n        - 'answers': a Dict in the SQuAD dataset format\n            {\n                'text': list of possible texts for the answer, as a list of strings\n                'answer_start': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    'exact_match': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\nExamples:\n\n    >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}]\n    >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}]\n    >>> squad_metric = datasets.load_metric(\"squad\")\n    >>> results = squad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 100.0, 'f1': 100.0}\n"






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,				_KWARGS_DESCRIPTION )
class 					__A    (    datasets.Metric ):

			'''simple docstring'''







			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return datasets.MetricInfo(
				    description=_DESCRIPTION       ,  citation=_CITATION       ,  inputs_description=_KWARGS_DESCRIPTION       ,  features=datasets.Features(
				        {
				            '''predictions''': {'''id''': datasets.Value('''string'''							), '''prediction_text''': datasets.Value('''string'''							)},
				            '''references''': {
				                '''id''': datasets.Value('''string'''							),
				                '''answers''': datasets.features.Sequence(
				                    {
				                        '''text''': datasets.Value('''string'''							),
				                        '''answer_start''': datasets.Value('''int32'''							),
				                    }							),
				            },
				        }							)       ,  codebase_urls=['''https://rajpurkar.github.io/SQuAD-explorer/''']       ,  reference_urls=['''https://rajpurkar.github.io/SQuAD-explorer/''']       ,  )





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		{prediction['''id''']: prediction['''prediction_text'''] for prediction in predictions}
				lowerCamelCase__								=		[
				    {
				        '''paragraphs''': [
				            {
				                '''qas''': [
				                    {
				                        '''answers''': [{'''text''': answer_text} for answer_text in ref['''answers''']['''text''']],
				                        '''id''': ref['''id'''],
				                    }
				                    for ref in references
				                ]
				            }
				        ]
				    }
				]
				lowerCamelCase__								=		evaluate(dataset=__lowerCAmelCase       ,  predictions=__lowerCAmelCase							)
				return score










import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
			import torch


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


_a 						=      logging.get_logger(__name__)
_a 						=      {"vocab_file": "spiece.model"}

_a 						=      {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_a 						=      {
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						VOCAB_FILES_NAMES
			lowerCAmelCase_							     =						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							     =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							     =						["""input_ids""", """attention_mask"""]





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase = None       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		{} if sp_model_kwargs is None else sp_model_kwargs

				lowerCamelCase__								=		kwargs.get('''name_or_path'''							)
				if name_or_path is None:
					logger.warning(
					    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
					    ''' you are testing the model, this can safely be ignored'''							)
					lowerCamelCase__								=		'''None'''

				# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
				lowerCamelCase__								=		'''<|endoftext|>''' if eos_token is None else eos_token
				lowerCamelCase__								=		'''<unk>''' if unk_token is None else unk_token
				if "gpt-sw3-7b" in name_or_path:
					lowerCamelCase__								=		unk_token if pad_token is None else pad_token
					lowerCamelCase__								=		eos_token if bos_token is None else bos_token
				else:
					lowerCamelCase__								=		'''<pad>''' if pad_token is None else pad_token
					lowerCamelCase__								=		'''<s>''' if bos_token is None else bos_token

				super().__init__(
				    do_lower_case=__lowerCAmelCase       ,  remove_space=__lowerCAmelCase       ,  keep_accents=__lowerCAmelCase       ,  bos_token=__lowerCAmelCase       ,  eos_token=__lowerCAmelCase       ,  unk_token=__lowerCAmelCase       ,  pad_token=__lowerCAmelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  **__lowerCAmelCase       ,  )

				lowerCamelCase__								=		do_lower_case
				lowerCamelCase__								=		remove_space
				lowerCamelCase__								=		keep_accents
				lowerCamelCase__								=		vocab_file

				lowerCamelCase__								=		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(__lowerCAmelCase							)

				# Used for whitespace normalization in input texts
				# fmt : off
				lowerCamelCase__								=		{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
				# fmt : on

				# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
				lowerCamelCase__								=		re.compile(
				    F'[{"".join(map(__lowerCAmelCase       ,  list(range(0       ,  9							)							) + list(range(1_1       ,  3_2							)							) + list(range(1_2_7       ,  1_6_0							)							) + [1_6_0, 1_7_3, 8_2_0_3]							)							)}]'							)





			def __getstate__(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.__dict__.copy()
				lowerCamelCase__								=		None
				return state





			def __setstate__(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		d

				# for backward compatibility
				if not hasattr(self       ,  '''sp_model_kwargs'''							):
					lowerCamelCase__								=		{}

				lowerCamelCase__								=		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(self.vocab_file							)





			@property
			# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return len(self.sp_model							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.non_printing_characters_re.sub(''''''       ,  __lowerCAmelCase							)

				# Normalize whitespaces
				lowerCamelCase__								=		''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]							)

				# NFC Unicode normalization
				lowerCamelCase__								=		unicodedata.normalize('''NFC'''       ,  __lowerCAmelCase							)
				return text





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.preprocess_text(__lowerCAmelCase							)
				return self.sp_model.encode(__lowerCAmelCase       ,  out_type=__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.PieceToId(__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.IdToPiece(__lowerCAmelCase							)





			@staticmethod
			def 		__lowerCamelCase				(    __lowerCAmelCase							):



				'''simple docstring'''







				return out_string





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[]
				lowerCamelCase__								=		''''''
				lowerCamelCase__								=		False
				for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
						if not prev_is_special:
							out_string += " "

						out_string += self.sp_model.decode(__lowerCAmelCase							) + token
						lowerCamelCase__								=		True
						lowerCamelCase__								=		[]
					else:
						current_sub_tokens.append(__lowerCAmelCase							)
						lowerCamelCase__								=		False
				out_string += self.sp_model.decode(__lowerCAmelCase							)

				return out_string





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		{self.convert_ids_to_tokens(__lowerCAmelCase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				if not os.path.isdir(__lowerCAmelCase							):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
					return
				lowerCamelCase__								=		os.path.join(
				    __lowerCAmelCase       ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

				if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCAmelCase							) and os.path.isfile(self.vocab_file							):
					copyfile(self.vocab_file       ,  __lowerCAmelCase							)
				elif not os.path.isfile(self.vocab_file							):
					with open(__lowerCAmelCase       ,  '''wb'''							) as fi:
						lowerCamelCase__								=		self.sp_model.serialized_model_proto()
						fi.write(__lowerCAmelCase							)

				return (out_vocab_file,)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = False							):



				'''simple docstring'''







				if isinstance(__lowerCAmelCase       ,  __lowerCAmelCase							):
					lowerCamelCase__								=		self.preprocess_text(__lowerCAmelCase							)
					lowerCamelCase__								=		self.sp_model.encode(__lowerCAmelCase							)
				else:
					lowerCamelCase__								=		[self.preprocess_text(__lowerCAmelCase							) for t in text]
					lowerCamelCase__								=		self.sp_model.encode(__lowerCAmelCase							)

				if return_tensors is True or return_tensors == "pt":
					lowerCamelCase__								=		torch.tensor(__lowerCAmelCase							)

				return token_ids





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.decode(__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
				lowerCamelCase__								=		(
				    F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(__lowerCAmelCase							) + F'{self.bos_token}Bot:'
				)
				return self.encode(text=__lowerCAmelCase							)



