




from typing import Dict, List

from nltk.translate import gleu_score

import datasets
from datasets import MetricInfo


lowercase__				:     Dict 		=						"\\n@misc{wu2016googles,\n      title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation},\n      author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey\n              and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin\n              Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto\n              Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and\n              Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes\n              and Jeffrey Dean},\n      year={2016},\n      eprint={1609.08144},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL}\n}\n"

lowercase__				:     Dict 		=						"\\nThe BLEU score has some undesirable properties when used for single\nsentences, as it was designed to be a corpus measure. We therefore\nuse a slightly different score for our RL experiments which we call\nthe 'GLEU score'. For the GLEU score, we record all sub-sequences of\n1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then\ncompute a recall, which is the ratio of the number of matching n-grams\nto the number of total n-grams in the target (ground truth) sequence,\nand a precision, which is the ratio of the number of matching n-grams\nto the number of total n-grams in the generated output sequence. Then\nGLEU score is simply the minimum of recall and precision. This GLEU\nscore's range is always between 0 (no matches) and 1 (all match) and\nit is symmetrical when switching output and target. According to\nour experiments, GLEU score correlates quite well with the BLEU\nmetric on a corpus level but does not have its drawbacks for our per\nsentence reward objective.\n"

lowercase__				:     List[str] 		=						"\\nComputes corpus-level Google BLEU (GLEU) score of translated segments against one or more references.\nInstead of averaging the sentence level GLEU scores (i.e. macro-average precision), Wu et al. (2016) sum up the matching\ntokens and the max of hypothesis and reference tokens for each sentence, then compute using the aggregate values.\n\nArgs:\n    predictions (list of str): list of translations to score.\n        Each translation should be tokenized into a list of tokens.\n    references (list of list of str): list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    min_len (int): The minimum order of n-gram this function should extract. Defaults to 1.\n    max_len (int): The maximum order of n-gram this function should extract. Defaults to 4.\n\nReturns:\n    'google_bleu': google_bleu score\n\nExamples:\n    Example 1:\n        >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',\n        ...         'ensures', 'that', 'the', 'rubber', 'duck', 'always',\n        ...         'disobeys', 'the', 'commands', 'of', 'the', 'cat']\n        >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which',\n        ...          'guarantees', 'the', 'rubber', 'duck', 'forces', 'never',\n        ...          'being', 'under', 'the', 'command', 'of', 'the', 'cat']\n\n        >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',\n        ...         'interested', 'in', 'world', 'history']\n        >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',\n        ...          'because', 'he', 'read', 'the', 'book']\n\n        >>> list_of_references = [[ref1a], [ref2a]]\n        >>> hypotheses = [hyp1, hyp2]\n        >>> google_bleu = datasets.load_metric(\"google_bleu\")\n        >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references)\n        >>> print(round(results[\"google_bleu\"], 2))\n        0.44\n\n    Example 2:\n        >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',\n        ...         'ensures', 'that', 'the', 'rubber', 'duck', 'always',\n        ...         'disobeys', 'the', 'commands', 'of', 'the', 'cat']\n        >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which',\n        ...          'guarantees', 'the', 'rubber', 'duck', 'forces', 'never',\n        ...          'being', 'under', 'the', 'command', 'of', 'the', 'cat']\n        >>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',\n        ...          'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never',\n        ...          'heed', 'the', 'cat', 'commands']\n        >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',\n        ...          'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions',\n        ...          'of', 'the', 'cat']\n\n        >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',\n        ...         'interested', 'in', 'world', 'history']\n        >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',\n        ...          'because', 'he', 'read', 'the', 'book']\n\n        >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]\n        >>> hypotheses = [hyp1, hyp2]\n        >>> google_bleu = datasets.load_metric(\"google_bleu\")\n        >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references)\n        >>> print(round(results[\"google_bleu\"], 2))\n        0.61\n\n    Example 3:\n        >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',\n        ...         'ensures', 'that', 'the', 'rubber', 'duck', 'always',\n        ...         'disobeys', 'the', 'commands', 'of', 'the', 'cat']\n        >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which',\n        ...          'guarantees', 'the', 'rubber', 'duck', 'forces', 'never',\n        ...          'being', 'under', 'the', 'command', 'of', 'the', 'cat']\n        >>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',\n        ...          'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never',\n        ...          'heed', 'the', 'cat', 'commands']\n        >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',\n        ...          'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions',\n        ...          'of', 'the', 'cat']\n\n        >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',\n        ...         'interested', 'in', 'world', 'history']\n        >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',\n        ...          'because', 'he', 'read', 'the', 'book']\n\n        >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]\n        >>> hypotheses = [hyp1, hyp2]\n        >>> google_bleu = datasets.load_metric(\"google_bleu\")\n        >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references, min_len=2)\n        >>> print(round(results[\"google_bleu\"], 2))\n        0.53\n\n    Example 4:\n        >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',\n        ...         'ensures', 'that', 'the', 'rubber', 'duck', 'always',\n        ...         'disobeys', 'the', 'commands', 'of', 'the', 'cat']\n        >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which',\n        ...          'guarantees', 'the', 'rubber', 'duck', 'forces', 'never',\n        ...          'being', 'under', 'the', 'command', 'of', 'the', 'cat']\n        >>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',\n        ...          'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never',\n        ...          'heed', 'the', 'cat', 'commands']\n        >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',\n        ...          'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions',\n        ...          'of', 'the', 'cat']\n\n        >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',\n        ...         'interested', 'in', 'world', 'history']\n        >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',\n        ...          'because', 'he', 'read', 'the', 'book']\n\n        >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]\n        >>> hypotheses = [hyp1, hyp2]\n        >>> google_bleu = datasets.load_metric(\"google_bleu\")\n        >>> results = google_bleu.compute(predictions=hypotheses,references=list_of_references, min_len=2, max_len=6)\n        >>> print(round(results[\"google_bleu\"], 2))\n        0.4\n"
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,   _KWARGS_DESCRIPTION			)
class 		UpperCAmelCase   (				datasets.Metric			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							return datasets.MetricInfo(
							    description=_DESCRIPTION       ,	citation=_CITATION       ,	inputs_description=_KWARGS_DESCRIPTION       ,	features=datasets.Features(
							        {
							            "predictions": datasets.Sequence(datasets.Value("string"       ,	id="token"						)       ,	id="sequence"						),
							            "references": datasets.Sequence(
							                datasets.Sequence(datasets.Value("string"       ,	id="token"						)       ,	id="sequence"						)       ,	id="references"						),
							        }						)       ,	)






						def        snake_case__						(						self							:     Any       ,	__lowercase							:     List[List[List[str]]]       ,	__lowercase							:     List[List[str]]       ,	__lowercase							:     int = 1       ,	__lowercase							:     int = 4       ,	):






							"""simple docstring"""






							return {
							    "google_bleu": gleu_score.corpus_gleu(
							        list_of_references=__lowercase       ,	hypotheses=__lowercase       ,	min_len=__lowercase       ,	max_len=__lowercase						)
							}






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__				:     Optional[Any] 		=						logging.get_logger(__name__)

lowercase__				:     List[str] 		=						"▁"

lowercase__				:     Union[str, Any] 		=						{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowercase__				:     List[Any] 		=						{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowercase__				:     Tuple 		=						{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowercase__				:     Optional[int] 		=						{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

lowercase__				:     Dict 		=						{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}
class 		UpperCAmelCase   (				UpperCAmelCase__			):







						'''simple docstring'''





						lowerCAmelCase_     		=   ["input_ids"]

						lowerCAmelCase_     		=   VOCAB_FILES_NAMES
						lowerCAmelCase_     		=   PRETRAINED_INIT_CONFIGURATION
						lowerCAmelCase_     		=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase_     		=   PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase_     		=   RESOURCE_FILES_NAMES






						def __init__(						self							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Tuple=None       ,	__lowercase							:     List[str]=False       ,	__lowercase							:     List[str]="utf8"       ,	__lowercase							:     Union[str, Any]="[UNK]"       ,	__lowercase							:     List[str]="[SEP]"       ,	__lowercase							:     Optional[Any]="[PAD]"       ,	__lowercase							:     Any="[CLS]"       ,	__lowercase							:     Any="[MASK]"       ,	__lowercase							:     Optional[Dict[str, Any]] = None       ,	**__lowercase							:     Tuple       ,	):






							"""simple docstring"""






							snake_case_		    =     {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=__lowercase       ,	unk_token=__lowercase       ,	sep_token=__lowercase       ,	pad_token=__lowercase       ,	cls_token=__lowercase       ,	mask_token=__lowercase       ,	vocab_file=__lowercase       ,	encoding=__lowercase       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**__lowercase       ,	)
							snake_case_		    =     do_lower_case
							snake_case_		    =     sentencepiece_model_ckpt
							snake_case_		    =     spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(__lowercase						)

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
								snake_case_		    =     self.load_vocab(filepath=__lowercase						)
							else:
								snake_case_		    =     {self.sp_model.id_to_piece(__lowercase						): id for id in range(self.sp_model.get_piece_size()						)}
							snake_case_		    =     {v: k for k, v in self.vocab.items()}






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Optional[int]						):






							"""simple docstring"""






							if text is None:
								return None

							snake_case_		    =     self.tokenize(__lowercase						)
							snake_case_    ,      snake_case_		    =     "", []

							for i, ch in enumerate(__lowercase						):
								if ch in self.SP_CHAR_MAPPING:
									snake_case_		    =     self.SP_CHAR_MAPPING.get(__lowercase						)
								else:
									snake_case_		    =     unicodedata.normalize("NFKC"       ,	__lowercase						)
								if self.is_whitespace(__lowercase						):
									continue
								normalized_text += ch
								char_mapping.extend([i] * len(__lowercase						)						)

							snake_case_    ,      snake_case_    ,      snake_case_		    =     normalized_text, [], 0

							if self.do_lower_case:
								snake_case_		    =     text.lower()

							for token in split_tokens:
								if token[:1] == "▁":
									snake_case_		    =     token[1:]
								snake_case_		    =     text[offset:].index(__lowercase						) + offset
								snake_case_		    =     start + len(__lowercase						)

								token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
								snake_case_		    =     end
							return token_mapping






						@property
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							return len(self.vocab						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							return dict(self.vocab       ,	**self.added_tokens_encoder						)






						def __getstate__(						self							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     self.__dict__.copy()
							snake_case_		    =     None
							return state






						def __setstate__(						self							:     str       ,	__lowercase							:     str						):






							"""simple docstring"""






							snake_case_		    =     d

							# for backward compatibility
							if not hasattr(self       ,	"sp_model_kwargs"						):
								snake_case_		    =     {}

							snake_case_		    =     spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.sentencepiece_model_ckpt						)






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							return "".join((self.SP_CHAR_MAPPING.get(__lowercase       ,	__lowercase						) for c in text)						)






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     int       ,	__lowercase							:     Any=False       ,	__lowercase							:     str=64       ,	__lowercase							:     Optional[Any]=0.1						):






							"""simple docstring"""






							if self.sp_model_kwargs.get("enable_sampling"						) is True:
								snake_case_		    =     True
							if self.sp_model_kwargs.get("alpha"						) is not None:
								snake_case_		    =     self.sp_model_kwargs.get("alpha"						)
							if self.sp_model_kwargs.get("nbest_size"						) is not None:
								snake_case_		    =     self.sp_model_kwargs.get("nbest_size"						)

							if not enable_sampling:
								snake_case_		    =     self.sp_model.EncodeAsPieces(__lowercase						)
							else:
								snake_case_		    =     self.sp_model.SampleEncodeAsPieces(__lowercase       ,	__lowercase       ,	__lowercase						)
							snake_case_		    =     []
							for pi, piece in enumerate(__lowercase						):
								if piece == SPIECE_UNDERLINE:
									if not pieces[pi + 1].startswith(__lowercase						) and pi != 0:
										new_pieces.append(__lowercase						)
										continue
									else:
										continue
								snake_case_		    =     0
								for i, chunk in enumerate(__lowercase						):
									if chunk == SPIECE_UNDERLINE:
										continue
									if self.is_ch_char(__lowercase						) or self.is_punct(__lowercase						):
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										new_pieces.append(__lowercase						)
										snake_case_		    =     i + 1
									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										snake_case_		    =     i
									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										snake_case_		    =     i
								if len(__lowercase						) > lst_i:
									new_pieces.append(piece[lst_i:]						)
							return new_pieces






						def        snake_case__						(						self							:     List[Any]       ,	__lowercase							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     "".join(__lowercase						).replace(__lowercase       ,	" "						).strip()
							return out_string






						def        snake_case__						(						self							:     int       ,	__lowercase							:     int						):






							"""simple docstring"""






							snake_case_		    =     self.convert_ids_to_tokens(__lowercase						)
							snake_case_		    =     "".join(__lowercase						).replace(__lowercase       ,	" "						).strip()
							return out_string






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Any						):






							"""simple docstring"""






							return self.vocab.get(__lowercase       ,	self.vocab.get(self.unk_token						)						)






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[Any]						):






							"""simple docstring"""






							return self.reverse_vocab.get(__lowercase       ,	self.unk_token						)






						def        snake_case__						(						self							:     Optional[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     int=None						):






							"""simple docstring"""






							if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							snake_case_		    =     [self.cls_token_id]
							snake_case_		    =     [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[str]       ,	__lowercase							:     Any=None						):






							"""simple docstring"""






							if offset_mapping_a is None:
								return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     List[Any]=None       ,	__lowercase							:     Dict=False						):






							"""simple docstring"""






							if already_has_special_tokens:
								if token_ids_a is not None:
									raise ValueError(
									    "You should not supply a second sequence if the provided sequence of "
									    "ids is already formatted with special tokens for the model."						)
								return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
								return [1] + ([0] * len(__lowercase						)) + [1, 1] + ([0] * len(__lowercase						)) + [1]
							return [1] + ([0] * len(__lowercase						)) + [1]






						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     List[int]       ,	__lowercase							:     Optional[List[int]] = None						):






							"""simple docstring"""






							if token_ids_a is None:
								# [CLS] X [SEP]
								return (len(__lowercase						) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(__lowercase						) + 1) + [1] * (len(__lowercase						) + 3)






						def        snake_case__						(						self							:     Any       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							if "\u4e00" <= char <= "\u9fff":
								return True
							return False






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     Any						):






							"""simple docstring"""






							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
								return True
							return False






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Dict						):






							"""simple docstring"""






							if char in ",;:.?!~，；：。？！《》【】":
								return True
							return False






						def        snake_case__						(						self							:     Union[str, Any]       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							if char == " " or char == "\t" or char == "\n" or char == "\r":
								return True
							if len(__lowercase						) == 1:
								snake_case_		    =     unicodedata.category(__lowercase						)
								if cat == "Zs":
									return True
							return False






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     {}
							with io.open(__lowercase       ,	"r"       ,	encoding="utf-8"						) as f:
								for index, line in enumerate(__lowercase						):
									snake_case_		    =     line.rstrip("\n"						)
									snake_case_		    =     int(__lowercase						)

							return token_to_idx






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     str       ,	__lowercase							:     Optional[str] = None						):






							"""simple docstring"""






							snake_case_		    =     0
							if os.path.isdir(__lowercase						):
								snake_case_		    =     os.path.join(
								    __lowercase       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
							else:
								snake_case_		    =     (filename_prefix + "-" if filename_prefix else "") + save_directory
							with open(__lowercase       ,	"w"       ,	encoding="utf-8"						) as writer:
								for token, token_index in sorted(self.vocab.items()       ,	key=lambda __lowercase						: kv[1]						):
									if index != token_index:
										logger.warning(
										    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
										    " Please check that the vocabulary is not corrupted!"						)
										snake_case_		    =     token_index
									writer.write(token + "\n"						)
									index += 1

							snake_case_		    =     os.path.join(__lowercase       ,	"sentencepiece.bpe.model"						)
							with open(__lowercase       ,	"wb"						) as fi:
								snake_case_		    =     self.sp_model.serialized_model_proto()
								fi.write(__lowercase						)

							return (vocab_file,)
