


import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase											=       get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCamelCase											=       5_00_03
lowerCamelCase											=       5_00_02




@require_sentencepiece
@require_tokenizers
class A  (   UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  int      =PLBartTokenizer
      UpperCamelCase__							:  Dict      =None
      UpperCamelCase__							:  Optional[Any]      =False






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						List[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase       :		Any		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )
             tokenizer.save_pretrained(self.tmpdirname )






      def        lowerCamelCase       (					self						:     List[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Optional[int]		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		str		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		List[Any]		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )

             _lowerCamelCase       :		str		=tokenizer.vocab_size
             _lowerCamelCase       :		List[str]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 4			,   lowercase_ )]

             self.assertListEqual(lowercase_			,   ['__java__', '__python__', '__en_XX__', '<mask>'] )

             _lowerCamelCase       :		Optional[Any]		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Dict		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )






      def        lowerCamelCase       (					self						:     str )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=PLBartTokenizer(lowercase_			,   language_codes='multi'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Any		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.vocab_size
             _lowerCamelCase       :		Optional[int]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 7			,   lowercase_ )]

             self.assertListEqual(
                 lowercase_			,   ['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__'] )
             _lowerCamelCase       :		int		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Any		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )




@require_torch
@require_sentencepiece
@require_tokenizers
class A  (   unittest.TestCase     ):
      UpperCamelCase__							:  List[Any]      ='uclanlp/plbart-python-en_XX'
      UpperCamelCase__							:  List[str]      =[
          'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
          'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
      ]
      UpperCamelCase__							:  Optional[int]      =[
          'Returns the maximum value of a b c.',
          'Sums the values of a b c.',
      ]
      UpperCamelCase__							:  str      =[
          134,
          5452,
          33460,
          33441,
          33463,
          33465,
          33463,
          33449,
          988,
          20,
          33456,
          19,
          33456,
          771,
          39,
          4258,
          889,
          3318,
          33441,
          33463,
          33465,
          33463,
          33449,
          2471,
          2,
          PYTHON_CODE,
      ]






      @classmethod
      def        lowerCamelCase       (					cls						:     str )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase       :		PLBartTokenizer		=PLBartTokenizer.from_pretrained(
                 cls.checkpoint_name			,   language_codes='base'			,   src_lang='python'			,   tgt_lang='en_XX' )
             _lowerCamelCase       :		Any		=1
             return cls






      def        lowerCamelCase       (					self						:     List[str] )					->						Any:
             """simple docstring"""




             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']			,   5_0001 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']			,   5_0002 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']			,   5_0003 )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Tuple:
             """simple docstring"""




             self.assertIn(lowercase_			,   self.tokenizer.all_special_ids )
             _lowerCamelCase       :		Dict		=[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
             _lowerCamelCase       :		Optional[int]		=self.tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_ )
             _lowerCamelCase       :		int		=self.tokenizer.decode(generated_ids[1:]			,   skip_special_tokens=lowercase_ )
             self.assertEqual(lowercase_			,   lowercase_ )
             self.assertNotIn(self.tokenizer.eos_token			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
             self.assertIsInstance(src_text[0]			,   lowercase_ )
             _lowerCamelCase       :		Tuple		=10
             _lowerCamelCase       :		Optional[int]		=self.tokenizer(lowercase_			,   max_length=lowercase_			,   truncation=lowercase_ ).input_ids[0]
             self.assertEqual(ids[-2]			,   2 )
             self.assertEqual(ids[-1]			,   lowercase_ )
             self.assertEqual(len(lowercase_ )			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Dict )					->						Optional[int]:
             """simple docstring"""




             self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__'] )			,   [5_0004, 5_0001] )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=tempfile.mkdtemp()
             _lowerCamelCase       :		Dict		=self.tokenizer.fairseq_tokens_to_ids
             self.tokenizer.save_pretrained(lowercase_ )
             _lowerCamelCase       :		Any		=PLBartTokenizer.from_pretrained(lowercase_ )
             self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,   lowercase_ )






      @require_torch
      def        lowerCamelCase       (					self						:     Optional[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=self.tokenizer(self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   return_tensors='pt' )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
             self.assertEqual(batch.input_ids[1][-2:].tolist()			,   [2, PYTHON_CODE] )
             self.assertEqual(batch.decoder_input_ids[1][0]			,   lowercase_ )
             self.assertEqual(batch.decoder_input_ids[1][-1]			,   2 )
             self.assertEqual(batch.labels[1][-2:].tolist()			,   [2, EN_CODE] )






      @require_torch
      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		int		=self.tokenizer(
                 self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=len(self.expected_src_tokens )			,   return_tensors='pt'			,   )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             self.assertIsInstance(lowercase_			,   lowercase_ )

             self.assertEqual((2, 26)			,   batch.input_ids.shape )
             self.assertEqual((2, 26)			,   batch.attention_mask.shape )
             _lowerCamelCase       :		List[Any]		=batch.input_ids.tolist()[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )
             self.assertEqual(2			,   batch.decoder_input_ids[0, -1] )  # EOS
             # Test that special tokens are reset
             self.assertEqual(self.tokenizer.prefix_tokens			,   [] )
             self.assertEqual(self.tokenizer.suffix_tokens			,   [self.tokenizer.eos_token_id, PYTHON_CODE] )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=self.tokenizer(self.src_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=3			,   return_tensors='pt' )
             _lowerCamelCase       :		Dict		=self.tokenizer(
                 text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=10			,   return_tensors='pt' )
             _lowerCamelCase       :		List[str]		=targets['input_ids']
             _lowerCamelCase       :		Optional[Any]		=shift_tokens_right(lowercase_			,   self.tokenizer.pad_token_id )

             self.assertEqual(batch.input_ids.shape[1]			,   3 )
             self.assertEqual(batch.decoder_input_ids.shape[1]			,   10 )






      @require_torch
      def        lowerCamelCase       (					self						:     Any )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Any		=self.tokenizer._build_translation_inputs(
                 'A test'			,   return_tensors='pt'			,   src_lang='en_XX'			,   tgt_lang='java' )

             self.assertEqual(
                 nested_simplify(lowercase_ )			,   {
                     # A, test, EOS, en_XX
                     'input_ids': [[150, 242, 2, 5_0003]],
                     'attention_mask': [[1, 1, 1, 1]],
                     # java
                     'forced_bos_token_id': 5_0001,
                 }			,   )






from sklearn.metrics import mean_squared_error

import datasets


lowerCamelCase											=       '\\n@article{scikit-learn,\n title={Scikit-learn: Machine Learning in {P}ython},\n author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n journal={Journal of Machine Learning Research},\n volume={12},\n pages={2825--2830},\n year={2011}\n}\n'

lowerCamelCase											=       '\\nMean Squared Error(MSE) is the average of the square of difference between the predicted\nand actual values.\n'


lowerCamelCase											=       '\nArgs:\n    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)\n        Estimated target values.\n    references: array-like of shape (n_samples,) or (n_samples, n_outputs)\n        Ground truth (correct) target values.\n    sample_weight: array-like of shape (n_samples,), default=None\n        Sample weights.\n    multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average"\n        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.\n\n        "raw_values" : Returns a full set of errors in case of multioutput input.\n\n        "uniform_average" : Errors of all outputs are averaged with uniform weight.\n\n    squared : bool, default=True\n        If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value.\n\nReturns:\n    mse : mean squared error.\nExamples:\n\n    >>> mse_metric = datasets.load_metric("mse")\n    >>> predictions = [2.5, 0.0, 2, 8]\n    >>> references = [3, -0.5, 2, 7]\n    >>> results = mse_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'mse\': 0.375}\n    >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False)\n    >>> print(rmse_result)\n    {\'mse\': 0.6123724356957945}\n\n    If you\'re using multi-dimensional lists, then set the config as follows :\n\n    >>> mse_metric = datasets.load_metric("mse", "multilist")\n    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]\n    >>> references = [[0, 2], [-1, 2], [8, -5]]\n    >>> results = mse_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'mse\': 0.7083333333333334}\n    >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput=\'raw_values\')\n    >>> print(results) # doctest: +NORMALIZE_WHITESPACE\n    {\'mse\': array([0.41666667, 1. ])}\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,							_KWARGS_DESCRIPTION     )
class A  (   datasets.Metric     ):






      def        lowerCamelCase       (					self						:     List[Any] )					->						List[str]:
             """simple docstring"""




             return datasets.MetricInfo(
                 description=_DESCRIPTION			,   citation=_CITATION			,   inputs_description=_KWARGS_DESCRIPTION			,   features=datasets.Features(self._get_feature_types() )			,   reference_urls=[
                     'https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html'
                 ]			,   )






      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[int]:
             """simple docstring"""




             if self.config_name == "multilist":
                    return {
                        "predictions": datasets.Sequence(datasets.Value('float' ) ),
                        "references": datasets.Sequence(datasets.Value('float' ) ),
                    }
             else:
                    return {
                        "predictions": datasets.Value('float' ),
                        "references": datasets.Value('float' ),
                    }






      def        lowerCamelCase       (					self						:     List[Any]			,   lowercase_						:     List[Any]			,   lowercase_						:     Tuple			,   lowercase_						:     List[str]=None			,   lowercase_						:     Union[str, Any]="uniform_average"			,   lowercase_						:     Tuple=True )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=mean_squared_error(
                 lowercase_			,   lowercase_			,   sample_weight=lowercase_			,   multioutput=lowercase_			,   squared=lowercase_ )

             return {"mse": mse}


