

"""simple docstring"""




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A	:       Union[str, Any]        =						logging.get_logger(__name__)
__A	:       List[str]        =						{'vocab_file': 'spiece.model'}

__A	:       Tuple        =						{
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
}

__A	:       int        =						{
    'google/fnet-base': 5_12,
    'google/fnet-large': 5_12,
}

__A	:       List[str]        =						'▁'





class      __UpperCamelCase					(  _A		):
		SCREAMING_SNAKE_CASE														=       VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE														=       PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE														=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE														=       ["input_ids", "token_type_ids"]


		def __init__(self					:						Dict	,	__SCREAMING_SNAKE_CASE					:						List[str]	,	__SCREAMING_SNAKE_CASE					:						Any=False	,	__SCREAMING_SNAKE_CASE					:						List[Any]=True	,	__SCREAMING_SNAKE_CASE					:						int=True	,	__SCREAMING_SNAKE_CASE					:						Tuple="<unk>"	,	__SCREAMING_SNAKE_CASE					:						List[str]="[SEP]"	,	__SCREAMING_SNAKE_CASE					:						int="<pad>"	,	__SCREAMING_SNAKE_CASE					:						int="[CLS]"	,	__SCREAMING_SNAKE_CASE					:						Union[str, Any]="[MASK]"	,	__SCREAMING_SNAKE_CASE					:						Optional[Dict[str, Any]] = None	,	**__SCREAMING_SNAKE_CASE					:						Dict	,	):
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							A     =   (
							    AddedToken(__SCREAMING_SNAKE_CASE	,	lstrip=__SCREAMING_SNAKE_CASE	,	rstrip=__SCREAMING_SNAKE_CASE	,	normalized=__SCREAMING_SNAKE_CASE)
							    if isinstance(__SCREAMING_SNAKE_CASE	,	__SCREAMING_SNAKE_CASE)
							    else mask_token
							)

							A     =   {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=__SCREAMING_SNAKE_CASE	,	remove_space=__SCREAMING_SNAKE_CASE	,	keep_accents=__SCREAMING_SNAKE_CASE	,	unk_token=__SCREAMING_SNAKE_CASE	,	sep_token=__SCREAMING_SNAKE_CASE	,	pad_token=__SCREAMING_SNAKE_CASE	,	cls_token=__SCREAMING_SNAKE_CASE	,	mask_token=__SCREAMING_SNAKE_CASE	,	sp_model_kwargs=self.sp_model_kwargs	,	**__SCREAMING_SNAKE_CASE	,	)

							A     =   do_lower_case
							A     =   remove_space
							A     =   keep_accents
							A     =   vocab_file

							A     =   spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(__SCREAMING_SNAKE_CASE)


		@property
		def    SCREAMING_SNAKE_CASE__  (self					:						List[str]):
							return len(self.sp_model)


		def    SCREAMING_SNAKE_CASE__  (self					:						Tuple):
							A     =   {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
							vocab.update(self.added_tokens_encoder)
							return vocab


		def __getstate__(self					:						int):
							A     =   self.__dict__.copy()
							A     =   None
							return state


		def __setstate__(self					:						List[Any]	,	__SCREAMING_SNAKE_CASE					:						List[str]):
							A     =   d

							# for backward compatibility
							if not hasattr(self	,	"sp_model_kwargs"):
												A     =   {}

							A     =   spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(self.vocab_file)


		def    SCREAMING_SNAKE_CASE__  (self					:						str	,	__SCREAMING_SNAKE_CASE					:						Any):
							if self.remove_space:
												A     =   " ".join(inputs.strip().split())
							else:
												A     =   inputs
							A     =   outputs.replace("``"	,	"\"").replace("''"	,	"\"")

							if not self.keep_accents:
												A     =   unicodedata.normalize("NFKD"	,	__SCREAMING_SNAKE_CASE)
												A     =   "".join([c for c in outputs if not unicodedata.combining(__SCREAMING_SNAKE_CASE)])
							if self.do_lower_case:
												A     =   outputs.lower()

							return outputs


		def    SCREAMING_SNAKE_CASE__  (self					:						Dict	,	__SCREAMING_SNAKE_CASE					:						str):
							A     =   self.preprocess_text(__SCREAMING_SNAKE_CASE)
							A     =   self.sp_model.encode(__SCREAMING_SNAKE_CASE	,	out_type=__SCREAMING_SNAKE_CASE)
							A     =   []
							for piece in pieces:
												if len(__SCREAMING_SNAKE_CASE) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
																	A     =   self.sp_model.EncodeAsPieces(piece[:-1].replace(__SCREAMING_SNAKE_CASE	,	""))
																	if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																						if len(cur_pieces[0]) == 1:
																											A     =   cur_pieces[1:]
																						else:
																											A     =   cur_pieces[0][1:]
																	cur_pieces.append(piece[-1])
																	new_pieces.extend(__SCREAMING_SNAKE_CASE)
												else:
																	new_pieces.append(__SCREAMING_SNAKE_CASE)

							return new_pieces


		def    SCREAMING_SNAKE_CASE__  (self					:						int	,	__SCREAMING_SNAKE_CASE					:						List[Any]):
							return self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE)


		def    SCREAMING_SNAKE_CASE__  (self					:						List[Any]	,	__SCREAMING_SNAKE_CASE					:						int):
							return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE)


		def    SCREAMING_SNAKE_CASE__  (self					:						Optional[Any]	,	__SCREAMING_SNAKE_CASE					:						List[Any]):
							A     =   []
							A     =   ""
							A     =   False
							for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																	if not prev_is_special:
																						out_string += " "
																	out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE) + token
																	A     =   True
																	A     =   []
												else:
																	current_sub_tokens.append(__SCREAMING_SNAKE_CASE)
																	A     =   False
							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE)
							return out_string.strip()


		def    SCREAMING_SNAKE_CASE__  (self					:						List[str]	,	__SCREAMING_SNAKE_CASE					:						List[int]	,	__SCREAMING_SNAKE_CASE					:						bool = False	,	__SCREAMING_SNAKE_CASE					:						bool = None	,	__SCREAMING_SNAKE_CASE					:						bool = True	,	**__SCREAMING_SNAKE_CASE					:						Tuple	,	):
							A     =   kwargs.pop("use_source_tokenizer"	,	__SCREAMING_SNAKE_CASE)

							A     =   self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE	,	skip_special_tokens=__SCREAMING_SNAKE_CASE)

							# To avoid mixing byte-level and unicode for byte-level BPT
							# we need to build string separately for added tokens and byte-level tokens
							# cf. https://github.com/huggingface/transformers/issues/1133
							A     =   []
							A     =   []
							for token in filtered_tokens:
												if skip_special_tokens and token in self.all_special_ids:
																	continue
												if token in self.added_tokens_encoder:
																	if current_sub_text:
																						sub_texts.append(self.convert_tokens_to_string(__SCREAMING_SNAKE_CASE))
																						A     =   []
																	sub_texts.append(__SCREAMING_SNAKE_CASE)
												else:
																	current_sub_text.append(__SCREAMING_SNAKE_CASE)
							if current_sub_text:
												sub_texts.append(self.convert_tokens_to_string(__SCREAMING_SNAKE_CASE))

							# Mimic the behavior of the Rust tokenizer:
							# No space after <unk>
							if spaces_between_special_tokens:
												A     =   re.sub(R"(<unk>) "	,	R"\1"	,	" ".join(__SCREAMING_SNAKE_CASE))
							else:
												A     =   "".join(__SCREAMING_SNAKE_CASE)

							A     =   (
							    clean_up_tokenization_spaces
							    if clean_up_tokenization_spaces is not None
							    else self.clean_up_tokenization_spaces
							)
							if clean_up_tokenization_spaces:
												A     =   self.clean_up_tokenization(__SCREAMING_SNAKE_CASE)
												return clean_text
							else:
												return text


		def    SCREAMING_SNAKE_CASE__  (self					:						Optional[int]	,	__SCREAMING_SNAKE_CASE					:						List[int]	,	__SCREAMING_SNAKE_CASE					:						Optional[List[int]] = None):
							A     =   [self.sep_token_id]
							A     =   [self.cls_token_id]
							if token_ids_a is None:
												return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep


		def    SCREAMING_SNAKE_CASE__  (self					:						List[str]	,	__SCREAMING_SNAKE_CASE					:						List[int]	,	__SCREAMING_SNAKE_CASE					:						Optional[List[int]] = None	,	__SCREAMING_SNAKE_CASE					:						bool = False):

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__SCREAMING_SNAKE_CASE	,	token_ids_a=__SCREAMING_SNAKE_CASE	,	already_has_special_tokens=__SCREAMING_SNAKE_CASE)

							if token_ids_a is not None:
												return [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]
							return [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]


		def    SCREAMING_SNAKE_CASE__  (self					:						List[Any]	,	__SCREAMING_SNAKE_CASE					:						List[int]	,	__SCREAMING_SNAKE_CASE					:						Optional[List[int]] = None):
							A     =   [self.sep_token_id]
							A     =   [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


		def    SCREAMING_SNAKE_CASE__  (self					:						Any	,	__SCREAMING_SNAKE_CASE					:						str	,	__SCREAMING_SNAKE_CASE					:						Optional[str] = None):
							if not os.path.isdir(__SCREAMING_SNAKE_CASE):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
												return
							A     =   os.path.join(
							    __SCREAMING_SNAKE_CASE	,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE) and os.path.isfile(self.vocab_file):
												copyfile(self.vocab_file	,	__SCREAMING_SNAKE_CASE)
							elif not os.path.isfile(self.vocab_file):
												with open(__SCREAMING_SNAKE_CASE	,	"wb") as fi:
																	A     =   self.sp_model.serialized_model_proto()
																	fi.write(__SCREAMING_SNAKE_CASE)

							return (out_vocab_file,)












import sacrebleu as scb
from packaging import version

import datasets


_SCREAMING_SNAKE_CASE   						=		"""\
@inproceedings{post-2018-call,
    title = \"A Call for Clarity in Reporting {BLEU} Scores\",
    author = \"Post, Matt\",
    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",
    month = oct,
    year = \"2018\",
    address = \"Belgium, Brussels\",
    publisher = \"Association for Computational Linguistics\",
    url = \"https://www.aclweb.org/anthology/W18-6319\",
    pages = \"186--191\",
}
"""

_SCREAMING_SNAKE_CASE   						=		"""\
SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.
Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.
It also knows all the standard test sets and handles downloading, processing, and tokenization for you.

See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information.
"""

_SCREAMING_SNAKE_CASE   						=		"""
Produces BLEU scores along with its sufficient statistics
from a source against one or more references.

Args:
    predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens.
    references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length).
    smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are:
        - `'none'`: no smoothing
        - `'floor'`: increment zero counts
        - `'add-k'`: increment num/denom by k for n>1
        - `'exp'`: exponential decay
    smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`).
    tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are:
        - `'none'`: No tokenization.
        - `'zh'`: Chinese tokenization.
        - `'13a'`: mimics the `mteval-v13a` script from Moses.
        - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses
        - `'char'`: Language-agnostic character-level tokenization.
        - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3).
    lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`.
    force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`.
    use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`.

Returns:
    'score': BLEU score,
    'counts': Counts,
    'totals': Totals,
    'precisions': Precisions,
    'bp': Brevity penalty,
    'sys_len': predictions length,
    'ref_len': reference length,

Examples:

    Example 1:
        >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]
        >>> references = [[\"hello there general kenobi\", \"hello there !\"], [\"foo bar foobar\", \"foo bar foobar\"]]
        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")
        >>> results = sacrebleu.compute(predictions=predictions, references=references)
        >>> print(list(results.keys()))
        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
        >>> print(round(results[\"score\"], 1))
        100.0

    Example 2:
        >>> predictions = [\"hello there general kenobi\",
        ...                 \"on our way to ankh morpork\"]
        >>> references = [[\"hello there general kenobi\", \"hello there !\"],
        ...                 [\"goodbye ankh morpork\", \"ankh morpork\"]]
        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")
        >>> results = sacrebleu.compute(predictions=predictions,
        ...                             references=references)
        >>> print(list(results.keys()))
        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
        >>> print(round(results[\"score\"], 1))
        39.8
"""
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,					_KWARGS_DESCRIPTION       )
class SCREAMING_SNAKE_CASE_    (       datasets.Metric       ):






						def UpperCAmelCase_					(   self   :   str						)				->     List[str]:







							"""simple docstring"""


							if version.parse(scb.__version__						) < version.parse('1.4.12'						):
								raise ImportWarning(
								    'To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n'
								    'You can install it with `pip install "sacrebleu>=1.4.12"`.'						)
							return datasets.MetricInfo(
							    description=_DESCRIPTION		, citation=_CITATION		, homepage='https://github.com/mjpost/sacreBLEU'		, inputs_description=_KWARGS_DESCRIPTION		, features=datasets.Features(
							        {
							            'predictions': datasets.Value('string'		, id='sequence'						),
							            'references': datasets.Sequence(datasets.Value('string'		, id='sequence'						)		, id='references'						),
							        }						)		, codebase_urls=['https://github.com/mjpost/sacreBLEU']		, reference_urls=[
							        'https://github.com/mjpost/sacreBLEU',
							        'https://en.wikipedia.org/wiki/BLEU',
							        'https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213',
							    ]		, )






						def UpperCAmelCase_					(   self   :   List[Any]		, _A   :   List[str]		, _A   :   Optional[Any]		, _A   :   Tuple="exp"		, _A   :   Tuple=None		, _A   :   Optional[Any]=False		, _A   :   Dict=False		, _A   :   List[str]=None		, _A   :   Dict=False		, )				->     Optional[Any]:







							"""simple docstring"""


							snake_case_		:      List[str]           =						len(references[0]						)
							if any(len(_A						) != references_per_prediction for refs in references						):
								raise ValueError('Sacrebleu requires the same number of references for each prediction'						)
							snake_case_		:      Optional[Any]           =						[[refs[i] for refs in references] for i in range(_A						)]
							snake_case_		:      Any           =						scb.corpus_bleu(
							    _A		, _A		, smooth_method=_A		, smooth_value=_A		, force=_A		, lowercase=_A		, use_effective_order=_A		, **({'tokenize': tokenize} if tokenize else {})		, )
							snake_case_		:      int           =						{
							    'score': output.score,
							    'counts': output.counts,
							    'totals': output.totals,
							    'precisions': output.precisions,
							    'bp': output.bp,
							    'sys_len': output.sys_len,
							    'ref_len': output.ref_len,
							}
							return output_dict

