
"""simple docstring"""





import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

_a														=		{
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

_a														=		{
    """vinai/phobert-base""": 256,
    """vinai/phobert-large""": 256,
}



def       lowerCamelCase__		(			__snake_case	)							->		int:


					"""simple docstring"""





					_UpperCamelCase							 =  set()
					_UpperCamelCase							 =  word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)	)
										_UpperCamelCase							 =  char

					_UpperCamelCase							 =  set(__snake_case	)
					return pairs






class     _UpperCAmelCase(    lowerCamelCase	):
		lowercase__						=      VOCAB_FILES_NAMES
		lowercase__						=      PRETRAINED_VOCAB_FILES_MAP
		lowercase__						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(					self				,		__a				,		__a				,		__a="<s>"				,		__a="</s>"				,		__a="</s>"				,		__a="<s>"				,		__a="<unk>"				,		__a="<pad>"				,		__a="<mask>"				,		**__a				,		)       ->       Optional[int]:






							'''simple docstring'''
							super().__init__(
							    bos_token=__a				,		eos_token=__a				,		unk_token=__a				,		sep_token=__a				,		cls_token=__a				,		pad_token=__a				,		mask_token=__a				,		**__a				,		)

							_UpperCamelCase							 =  vocab_file
							_UpperCamelCase							 =  merges_file

							_UpperCamelCase							 =  {}
							_UpperCamelCase							 =  0
							_UpperCamelCase							 =  1
							_UpperCamelCase							 =  2
							_UpperCamelCase							 =  3

							self.add_from_file(__a)

							_UpperCamelCase							 =  {v: k for k, v in self.encoder.items()}

							with open(__a				,		encoding='''utf-8''') as merges_handle:
												_UpperCamelCase							 =  merges_handle.read().split('''\n''')[:-1]
							_UpperCamelCase							 =  [tuple(merge.split()[:-1]) for merge in merges]
							_UpperCamelCase							 =  dict(zip(__a				,		range(len(__a))))
							_UpperCamelCase							 =  {}



		def 						UpperCAmelCase							(					self				,		__a				,		__a = None)       ->       List[int]:






							'''simple docstring'''

							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_UpperCamelCase							 =  [self.cls_token_id]
							_UpperCamelCase							 =  [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep



		def 						UpperCAmelCase							(					self				,		__a				,		__a = None				,		__a = False)       ->       List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a				,		token_ids_a=__a				,		already_has_special_tokens=__a)

							if token_ids_a is None:
												return [1] + ([0] * len(__a)) + [1]
							return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a)) + [1]



		def 						UpperCAmelCase							(					self				,		__a				,		__a = None)       ->       List[int]:






							'''simple docstring'''
							_UpperCamelCase							 =  [self.sep_token_id]
							_UpperCamelCase							 =  [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



		@property
		def 						UpperCAmelCase							(					self)       ->       Tuple:






							'''simple docstring'''
							return len(self.encoder)



		def 						UpperCAmelCase							(					self)       ->       Tuple:






							'''simple docstring'''
							return dict(self.encoder				,		**self.added_tokens_encoder)



		def 						UpperCAmelCase							(					self				,		__a)       ->       str:






							'''simple docstring'''
							if token in self.cache:
												return self.cache[token]
							_UpperCamelCase							 =  tuple(__a)
							_UpperCamelCase							 =  tuple(list(word[:-1]) + [word[-1] + '''</w>'''])
							_UpperCamelCase							 =  get_pairs(__a)

							if not pairs:
												return token

							while True:
												_UpperCamelCase							 =  min(__a				,		key=lambda __a: self.bpe_ranks.get(__a				,		float('''inf''')))
												if bigram not in self.bpe_ranks:
																	break
												_UpperCamelCase ,  _UpperCamelCase							 =  bigram
												_UpperCamelCase							 =  []
												_UpperCamelCase							 =  0
												while i < len(__a):
																	try:
																						_UpperCamelCase							 =  word.index(__a				,		__a)
																	except ValueError:
																						new_word.extend(word[i:])
																						break
																	else:
																						new_word.extend(word[i:j])
																						_UpperCamelCase							 =  j

																	if word[i] == first and i < len(__a) - 1 and word[i + 1] == second:
																						new_word.append(first + second)
																						i += 2
																	else:
																						new_word.append(word[i])
																						i += 1
												_UpperCamelCase							 =  tuple(__a)
												_UpperCamelCase							 =  new_word
												if len(__a) == 1:
																	break
												else:
																	_UpperCamelCase							 =  get_pairs(__a)
							_UpperCamelCase							 =  '''@@ '''.join(__a)
							_UpperCamelCase							 =  word[:-4]
							_UpperCamelCase							 =  word
							return word



		def 						UpperCAmelCase							(					self				,		__a)       ->       Any:






							'''simple docstring'''
							_UpperCamelCase							 =  []

							_UpperCamelCase							 =  re.findall(R'''\S+\n?'''				,		__a)

							for token in words:
												split_tokens.extend(list(self.bpe(__a).split(''' ''')))
							return split_tokens



		def 						UpperCAmelCase							(					self				,		__a)       ->       List[str]:






							'''simple docstring'''
							return self.encoder.get(__a				,		self.encoder.get(self.unk_token))



		def 						UpperCAmelCase							(					self				,		__a)       ->       Union[str, Any]:






							'''simple docstring'''
							return self.decoder.get(__a				,		self.unk_token)



		def 						UpperCAmelCase							(					self				,		__a)       ->       List[Any]:






							'''simple docstring'''
							_UpperCamelCase							 =  ''' '''.join(__a).replace('''@@ '''				,		'''''').strip()
							return out_string



		def 						UpperCAmelCase							(					self				,		__a				,		__a = None)       ->       Tuple[str]:






							'''simple docstring'''
							if not os.path.isdir(__a):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
												return
							_UpperCamelCase							 =  os.path.join(
							    __a				,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
							_UpperCamelCase							 =  os.path.join(
							    __a				,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__a):
												copyfile(self.vocab_file				,		__a)

							if os.path.abspath(self.merges_file) != os.path.abspath(__a):
												copyfile(self.merges_file				,		__a)

							return out_vocab_file, out_merge_file




		def 						UpperCAmelCase							(					self				,		__a)       ->       List[str]:






							'''simple docstring'''
							if isinstance(__a				,		__a):
												try:
																	with open(__a				,		'''r'''				,		encoding='''utf-8''') as fd:
																						self.add_from_file(__a)
												except FileNotFoundError as fnfe:
																	raise fnfe
												except UnicodeError:
																	raise Exception(F'''Incorrect encoding detected in {f}, please rebuild the dataset''')
												return

							_UpperCamelCase							 =  f.readlines()
							for lineTmp in lines:
												_UpperCamelCase							 =  lineTmp.strip()
												_UpperCamelCase							 =  line.rfind(''' ''')
												if idx == -1:
																	raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\'''')
												_UpperCamelCase							 =  line[:idx]
												_UpperCamelCase							 =  len(self.encoder)








"""simple docstring"""





from sklearn.metrics import matthews_corrcoef

import datasets


_a														=		"""
Compute the Matthews correlation coefficient (MCC)

The Matthews correlation coefficient is used in machine learning as a
measure of the quality of binary and multiclass classifications. It takes
into account true and false positives and negatives and is generally
regarded as a balanced measure which can be used even if the classes are of
very different sizes. The MCC is in essence a correlation coefficient value
between -1 and +1. A coefficient of +1 represents a perfect prediction, 0
an average random prediction and -1 an inverse prediction.  The statistic
is also known as the phi coefficient. [source: Wikipedia]
"""

_a														=		"""
Args:
    predictions (list of int): Predicted labels, as returned by a model.
    references (list of int): Ground truth labels.
    sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`.
Returns:
    matthews_correlation (dict containing float): Matthews correlation.
Examples:
    Example 1, a basic example with only predictions and references as inputs:
        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")
        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],
        ...                                     predictions=[1, 2, 2, 0, 3, 3])
        >>> print(round(results['matthews_correlation'], 2))
        0.54

    Example 2, the same example as above, but also including sample weights:
        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")
        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],
        ...                                     predictions=[1, 2, 2, 0, 3, 3],
        ...                                     sample_weight=[0.5, 3, 1, 1, 1, 2])
        >>> print(round(results['matthews_correlation'], 2))
        0.1

    Example 3, the same example as above, but with sample weights that cause a negative correlation:
        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")
        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],
        ...                                     predictions=[1, 2, 2, 0, 3, 3],
        ...                                     sample_weight=[0.5, 1, 0, 0, 0, 1])
        >>> print(round(results['matthews_correlation'], 2))
        -0.25
"""

_a														=		"""\
@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}
"""






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,     _KWARGS_DESCRIPTION	)
class     _UpperCAmelCase(    datasets.Metric	):



		def 						UpperCAmelCase							(					self)       ->       List[Any]:






							'''simple docstring'''
							return datasets.MetricInfo(
							    description=_DESCRIPTION				,		citation=_CITATION				,		inputs_description=_KWARGS_DESCRIPTION				,		features=datasets.Features(
							        {
							            '''predictions''': datasets.Value('''int32'''),
							            '''references''': datasets.Value('''int32'''),
							        })				,		reference_urls=[
							        '''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html'''
							    ]				,		)




		def 						UpperCAmelCase							(					self				,		__a				,		__a				,		__a=None)       ->       Dict:






							'''simple docstring'''
							return {
							    "matthews_correlation": float(matthews_corrcoef(__a				,		__a				,		sample_weight=__a)),
							}






