






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
		import torch


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


UpperCamelCase 				=	logging.get_logger(__name__)
UpperCamelCase 				=	{"vocab_file": "spiece.model"}

UpperCamelCase 				=	{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

UpperCamelCase 				=	{
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}



class       __UpperCAmelCase		(lowercase__    ):
			__snake_case				:      List[Any] 						=							VOCAB_FILES_NAMES
			__snake_case				:      Optional[Any] 						=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case				:      int 						=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case				:      List[Any] 						=							["input_ids", "attention_mask"]







			def __init__(	self:  List[str]      ,	UpperCAmelCase_:  Optional[Any]      ,	UpperCAmelCase_:  Tuple=False      ,	UpperCAmelCase_:  int=False      ,	UpperCAmelCase_:  str=False      ,	UpperCAmelCase_:  Optional[Any]=None      ,	UpperCAmelCase_:  Optional[int]=None      ,	UpperCAmelCase_:  Union[str, Any]=None      ,	UpperCAmelCase_:  List[str]=None      ,	UpperCAmelCase_:  Optional[Dict[str, Any]] = None      ,	**UpperCAmelCase_:  Tuple      ,	):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  {} if sp_model_kwargs is None else sp_model_kwargs

									_SCREAMING_SNAKE_CASE	      =  kwargs.get("""name_or_path"""     )
									if name_or_path is None:
															logger.warning(
															    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
															    """ you are testing the model, this can safely be ignored"""     )
															_SCREAMING_SNAKE_CASE	      =  "None"

									# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
									_SCREAMING_SNAKE_CASE	      =  "<|endoftext|>" if eos_token is None else eos_token
									_SCREAMING_SNAKE_CASE	      =  "<unk>" if unk_token is None else unk_token
									if "gpt-sw3-7b" in name_or_path:
															_SCREAMING_SNAKE_CASE	      =  unk_token if pad_token is None else pad_token
															_SCREAMING_SNAKE_CASE	      =  eos_token if bos_token is None else bos_token
									else:
															_SCREAMING_SNAKE_CASE	      =  "<pad>" if pad_token is None else pad_token
															_SCREAMING_SNAKE_CASE	      =  "<s>" if bos_token is None else bos_token

									super().__init__(
									    do_lower_case=__lowerCamelCase      ,	remove_space=__lowerCamelCase      ,	keep_accents=__lowerCamelCase      ,	bos_token=__lowerCamelCase      ,	eos_token=__lowerCamelCase      ,	unk_token=__lowerCamelCase      ,	pad_token=__lowerCamelCase      ,	sp_model_kwargs=self.sp_model_kwargs      ,	**__lowerCamelCase      ,	)

									_SCREAMING_SNAKE_CASE	      =  do_lower_case
									_SCREAMING_SNAKE_CASE	      =  remove_space
									_SCREAMING_SNAKE_CASE	      =  keep_accents
									_SCREAMING_SNAKE_CASE	      =  vocab_file

									_SCREAMING_SNAKE_CASE	      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
									self.sp_model.Load(__lowerCamelCase     )

									# Used for whitespace normalization in input texts
									# fmt : off
									_SCREAMING_SNAKE_CASE	      =  {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
									# fmt : on

									# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
									_SCREAMING_SNAKE_CASE	      =  re.compile(
									    F'[{"".join(map(__lowerCamelCase      ,	list(range(0      ,	9     )     ) + list(range(11      ,	32     )     ) + list(range(127      ,	160     )     ) + [160, 173, 8_203]     )     )}]'     )







			def __getstate__(	self:  Tuple     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  self.__dict__.copy()
									_SCREAMING_SNAKE_CASE	      =  None
									return state







			def __setstate__(	self:  Dict      ,	UpperCAmelCase_:  Optional[Any]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  d

									# for backward compatibility
									if not hasattr(self      ,	"""sp_model_kwargs"""     ):
															_SCREAMING_SNAKE_CASE	      =  {}

									_SCREAMING_SNAKE_CASE	      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
									self.sp_model.Load(self.vocab_file     )







			@property
			# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
			def 							UpperCamelCase      (	self:  Dict     ):







									'''simple docstring'''




									return len(self.sp_model     )







			def 							UpperCamelCase      (	self:  Optional[int]      ,	UpperCAmelCase_:  str     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  self.non_printing_characters_re.sub(""""""      ,	__lowerCamelCase     )

									# Normalize whitespaces
									_SCREAMING_SNAKE_CASE	      =  "".join([char if char not in self.whitespaces else """ """ for char in text]     )

									# NFC Unicode normalization
									_SCREAMING_SNAKE_CASE	      =  unicodedata.normalize("""NFC"""      ,	__lowerCamelCase     )
									return text







			def 							UpperCamelCase      (	self:  List[Any]      ,	UpperCAmelCase_:  str      ,	**UpperCAmelCase_:  List[Any]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  self.preprocess_text(__lowerCamelCase     )
									return self.sp_model.encode(__lowerCamelCase      ,	out_type=__lowerCamelCase     )







			def 							UpperCamelCase      (	self:  Any      ,	UpperCAmelCase_:  str     ):







									'''simple docstring'''




									return self.sp_model.PieceToId(__lowerCamelCase     )







			def 							UpperCamelCase      (	self:  Any      ,	UpperCAmelCase_:  int     ):







									'''simple docstring'''




									return self.sp_model.IdToPiece(__lowerCamelCase     )







			@staticmethod
			def 							UpperCamelCase      (	UpperCAmelCase_:  str     ):







									'''simple docstring'''




									return out_string







			def 							UpperCamelCase      (	self:  Optional[int]      ,	UpperCAmelCase_:  List[str]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  []
									_SCREAMING_SNAKE_CASE	      =  ""
									_SCREAMING_SNAKE_CASE	      =  False
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																					if not prev_is_special:
																											out_string += " "

																					out_string += self.sp_model.decode(__lowerCamelCase     ) + token
																					_SCREAMING_SNAKE_CASE	      =  True
																					_SCREAMING_SNAKE_CASE	      =  []
															else:
																					current_sub_tokens.append(__lowerCamelCase     )
																					_SCREAMING_SNAKE_CASE	      =  False
									out_string += self.sp_model.decode(__lowerCamelCase     )

									return out_string







			def 							UpperCamelCase      (	self:  str     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  {self.convert_ids_to_tokens(__lowerCamelCase     ): i for i in range(self.vocab_size     )}
									vocab.update(self.added_tokens_encoder     )
									return vocab







			def 							UpperCamelCase      (	self:  str      ,	UpperCAmelCase_:  str      ,	UpperCAmelCase_:  Optional[str] = None     ):







									'''simple docstring'''




									if not os.path.isdir(__lowerCamelCase     ):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory'     )
															return
									_SCREAMING_SNAKE_CASE	      =  os.path.join(
									    __lowerCamelCase      ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

									if os.path.abspath(self.vocab_file     ) != os.path.abspath(__lowerCamelCase     ) and os.path.isfile(self.vocab_file     ):
															copyfile(self.vocab_file      ,	__lowerCamelCase     )
									elif not os.path.isfile(self.vocab_file     ):
															with open(__lowerCamelCase      ,	"""wb"""     ) as fi:
																					_SCREAMING_SNAKE_CASE	      =  self.sp_model.serialized_model_proto()
																					fi.write(__lowerCamelCase     )

									return (out_vocab_file,)







			def 							UpperCamelCase      (	self:  Optional[int]      ,	UpperCAmelCase_:  Union[str, List[str]]      ,	UpperCAmelCase_:  Union[str, bool] = False     ):







									'''simple docstring'''




									if isinstance(__lowerCamelCase      ,	__lowerCamelCase     ):
															_SCREAMING_SNAKE_CASE	      =  self.preprocess_text(__lowerCamelCase     )
															_SCREAMING_SNAKE_CASE	      =  self.sp_model.encode(__lowerCamelCase     )
									else:
															_SCREAMING_SNAKE_CASE	      =  [self.preprocess_text(__lowerCamelCase     ) for t in text]
															_SCREAMING_SNAKE_CASE	      =  self.sp_model.encode(__lowerCamelCase     )

									if return_tensors is True or return_tensors == "pt":
															_SCREAMING_SNAKE_CASE	      =  torch.tensor(__lowerCamelCase     )

									return token_ids







			def 							UpperCamelCase      (	self:  Tuple      ,	UpperCAmelCase_:  Union[int, List[int]]     ):







									'''simple docstring'''




									return self.sp_model.decode(__lowerCamelCase     )







			def 							UpperCamelCase      (	self:  Any      ,	UpperCAmelCase_:  "Conversation"     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  [F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
									_SCREAMING_SNAKE_CASE	      =  (
									    F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(__lowerCamelCase     ) + F'{self.bos_token}Bot:'
									)
									return self.encode(text=__lowerCamelCase     )





import datasets

from .evaluate import evaluate


A      :    Dict		=					"\\n@article{hendrycks2021cuad,\n      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},\n      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},\n      journal={arXiv preprint arXiv:2103.06268},\n      year={2021}\n}\n"

A      :    Any		=					"\nThis metric wrap the official scoring script for version 1 of the Contract\nUnderstanding Atticus Dataset (CUAD).\nContract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510\ncommercial legal contracts that have been manually labeled to identify 41 categories of important\nclauses that lawyers look for when reviewing contracts in connection with corporate transactions.\n"

A      :    str		=					"\nComputes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair as given in the references (see below)\n        - 'prediction_text': list of possible texts for the answer, as a list of strings\n        depending on a threshold on the confidence probability of each prediction.\n    references: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair (see above),\n        - 'answers': a Dict in the CUAD dataset format\n            {\n                'text': list of possible texts for the answer, as a list of strings\n                'answer_start': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    'exact_match': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\n    'aupr': Area Under the Precision-Recall curve\n    'prec_at_80_recall': Precision at 80% recall\n    'prec_at_90_recall': Precision at 90% recall\nExamples:\n    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]\n    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]\n    >>> cuad_metric = datasets.load_metric(\"cuad\")\n    >>> results = cuad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}\n"







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,  _KWARGS_DESCRIPTION)
class 			_lowercase (							datasets.Metric):


       """simple docstring"""




       def 						lowerCAmelCase     (   self							:			Optional[Any]       ):


             '''simple docstring'''



             return datasets.MetricInfo(
                 description=_DESCRIPTION					,      citation=_CITATION					,      inputs_description=_KWARGS_DESCRIPTION					,      features=datasets.Features(
                     {
                         "predictions": {
                             "id": datasets.Value("string"       ),
                             "prediction_text": datasets.features.Sequence(datasets.Value("string"       )       ),
                         },
                         "references": {
                             "id": datasets.Value("string"       ),
                             "answers": datasets.features.Sequence(
                                 {
                                     "text": datasets.Value("string"       ),
                                     "answer_start": datasets.Value("int32"       ),
                                 }       ),
                         },
                     }       )					,      codebase_urls=["https://www.atticusprojectai.org/cuad"]					,      reference_urls=["https://www.atticusprojectai.org/cuad"]					,      )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]					,      __lowerCamelCase							:			List[str]					,      __lowerCamelCase							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				int    =  {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
             lowerCamelCase__		:				Any    =  [
                 {
                     "paragraphs": [
                         {
                             "qas": [
                                 {
                                     "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
                                     "id": ref["id"],
                                 }
                                 for ref in references
                             ]
                         }
                     ]
                 }
             ]
             lowerCamelCase__		:				Optional[int]    =  evaluate(dataset=__lowerCamelCase					,      predictions=__lowerCamelCase       )
             return score


