


import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


lowerCamelCase											=       '\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

lowerCamelCase											=       '\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

lowerCamelCase											=       '\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'




def 	a_      (    SCREAMING_SNAKE_CASE__					:      Optional[int]		,			SCREAMING_SNAKE_CASE__					:      Tuple     ):
       '''simple docstring'''





       return float((preds == labels).mean()     )




def 	a_      (    SCREAMING_SNAKE_CASE__					:      Optional[int]		,			SCREAMING_SNAKE_CASE__					:      Optional[int]     ):
       '''simple docstring'''





       _lowerCamelCase       :		List[str]		=simple_accuracy(SCREAMING_SNAKE_CASE__		,			SCREAMING_SNAKE_CASE__     )
       _lowerCamelCase       :		Any		=float(fa_score(y_true=SCREAMING_SNAKE_CASE__		,			y_pred=SCREAMING_SNAKE_CASE__     )     )
       return {
           "accuracy": acc,
           "f1": fa,
       }




def 	a_      (    SCREAMING_SNAKE_CASE__					:      Optional[int]		,			SCREAMING_SNAKE_CASE__					:      List[Any]     ):
       '''simple docstring'''





       _lowerCamelCase       :		List[Any]		=np.array(SCREAMING_SNAKE_CASE__     )
       _lowerCamelCase       :		str		=np.array(SCREAMING_SNAKE_CASE__     )
       _lowerCamelCase       :		Union[str, Any]		=en_sentvecs.shape[0]

       # mean centering
       _lowerCamelCase       :		Optional[Any]		=en_sentvecs - np.mean(SCREAMING_SNAKE_CASE__		,			axis=0     )
       _lowerCamelCase       :		Any		=in_sentvecs - np.mean(SCREAMING_SNAKE_CASE__		,			axis=0     )

       _lowerCamelCase       :		Optional[int]		=cdist(SCREAMING_SNAKE_CASE__		,			SCREAMING_SNAKE_CASE__		,			'cosine'     )
       _lowerCamelCase       :		int		=np.array(range(SCREAMING_SNAKE_CASE__     )     )
       _lowerCamelCase       :		Optional[int]		=sim.argsort(axis=1     )[:, :10]
       _lowerCamelCase       :		List[str]		=np.any(preds == actual[:, None]		,			axis=1     )
       return float(matches.mean()     )




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,							_KWARGS_DESCRIPTION     )
class A  (   datasets.Metric     ):






      def        lowerCamelCase       (					self						:     Any )					->						Any:
             """simple docstring"""




             if self.config_name not in [
                 "wnli",
                 "copa",
                 "sna",
                 "csqa",
                 "wstp",
                 "inltkh",
                 "bbca",
                 "cvit-mkb-clsr",
                 "iitp-mr",
                 "iitp-pr",
                 "actsa-sc",
                 "md",
                 "wiki-ner",
             ]:
                    raise KeyError(
                        'You should supply a configuration name selected in '
                        '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
                        '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
                        '"wiki-ner"]' )
             return datasets.MetricInfo(
                 description=_DESCRIPTION			,   citation=_CITATION			,   inputs_description=_KWARGS_DESCRIPTION			,   features=datasets.Features(
                     {
                         'predictions': datasets.Value('int64' )
                         if self.config_name != 'cvit-mkb-clsr'
                         else datasets.Sequence(datasets.Value('float32' ) ),
                         'references': datasets.Value('int64' )
                         if self.config_name != 'cvit-mkb-clsr'
                         else datasets.Sequence(datasets.Value('float32' ) ),
                     } )			,   codebase_urls=[]			,   reference_urls=[]			,   format='numpy' if self.config_name != 'cvit-mkb-clsr' else None			,   )






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     Optional[Any]			,   lowercase_						:     Tuple )					->						Optional[Any]:
             """simple docstring"""




             if self.config_name == "cvit-mkb-clsr":
                    return {"precision@10": precision_at_aa(lowercase_			,   lowercase_ )}
             elif self.config_name in ["wiki-ner"]:
                    return acc_and_fa(lowercase_			,   lowercase_ )
             elif self.config_name in [
                 "wnli",
                 "copa",
                 "sna",
                 "csqa",
                 "wstp",
                 "inltkh",
                 "bbca",
                 "iitp-mr",
                 "iitp-pr",
                 "actsa-sc",
                 "md",
             ]:
                    return {"accuracy": simple_accuracy(lowercase_			,   lowercase_ )}
             else:
                    raise KeyError(
                        'You should supply a configuration name selected in '
                        '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
                        '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
                        '"wiki-ner"]' )






import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase											=       get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCamelCase											=       5_00_03
lowerCamelCase											=       5_00_02




@require_sentencepiece
@require_tokenizers
class A  (   UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  int      =PLBartTokenizer
      UpperCamelCase__							:  Dict      =None
      UpperCamelCase__							:  Optional[Any]      =False






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						List[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase       :		Any		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )
             tokenizer.save_pretrained(self.tmpdirname )






      def        lowerCamelCase       (					self						:     List[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=PLBartTokenizer(lowercase_			,   language_codes='base'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Optional[int]		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		str		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		List[Any]		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )

             _lowerCamelCase       :		str		=tokenizer.vocab_size
             _lowerCamelCase       :		List[str]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 4			,   lowercase_ )]

             self.assertListEqual(lowercase_			,   ['__java__', '__python__', '__en_XX__', '<mask>'] )

             _lowerCamelCase       :		Optional[Any]		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Dict		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )






      def        lowerCamelCase       (					self						:     str )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=PLBartTokenizer(lowercase_			,   language_codes='multi'			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Any		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]			,   )

             _lowerCamelCase       :		Optional[Any]		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '<unk>',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '<unk>',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Dict		=tokenizer.vocab_size
             _lowerCamelCase       :		Optional[int]		=[tokenizer.convert_ids_to_tokens(lowercase_ ) for x in range(end - 7			,   lowercase_ )]

             self.assertListEqual(
                 lowercase_			,   ['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__'] )
             _lowerCamelCase       :		int		='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
             _lowerCamelCase       :		Any		=tokenizer(lowercase_ ).input_ids
             self.assertEqual(
                 tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_			,   clean_up_tokenization_spaces=lowercase_ )			,   lowercase_			,   )




@require_torch
@require_sentencepiece
@require_tokenizers
class A  (   unittest.TestCase     ):
      UpperCamelCase__							:  List[Any]      ='uclanlp/plbart-python-en_XX'
      UpperCamelCase__							:  List[str]      =[
          'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
          'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
      ]
      UpperCamelCase__							:  Optional[int]      =[
          'Returns the maximum value of a b c.',
          'Sums the values of a b c.',
      ]
      UpperCamelCase__							:  str      =[
          134,
          5452,
          33460,
          33441,
          33463,
          33465,
          33463,
          33449,
          988,
          20,
          33456,
          19,
          33456,
          771,
          39,
          4258,
          889,
          3318,
          33441,
          33463,
          33465,
          33463,
          33449,
          2471,
          2,
          PYTHON_CODE,
      ]






      @classmethod
      def        lowerCamelCase       (					cls						:     str )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase       :		PLBartTokenizer		=PLBartTokenizer.from_pretrained(
                 cls.checkpoint_name			,   language_codes='base'			,   src_lang='python'			,   tgt_lang='en_XX' )
             _lowerCamelCase       :		Any		=1
             return cls






      def        lowerCamelCase       (					self						:     List[str] )					->						Any:
             """simple docstring"""




             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']			,   5_0001 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']			,   5_0002 )
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']			,   5_0003 )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Tuple:
             """simple docstring"""




             self.assertIn(lowercase_			,   self.tokenizer.all_special_ids )
             _lowerCamelCase       :		Dict		=[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
             _lowerCamelCase       :		Optional[int]		=self.tokenizer.decode(lowercase_			,   skip_special_tokens=lowercase_ )
             _lowerCamelCase       :		int		=self.tokenizer.decode(generated_ids[1:]			,   skip_special_tokens=lowercase_ )
             self.assertEqual(lowercase_			,   lowercase_ )
             self.assertNotIn(self.tokenizer.eos_token			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
             self.assertIsInstance(src_text[0]			,   lowercase_ )
             _lowerCamelCase       :		Tuple		=10
             _lowerCamelCase       :		Optional[int]		=self.tokenizer(lowercase_			,   max_length=lowercase_			,   truncation=lowercase_ ).input_ids[0]
             self.assertEqual(ids[-2]			,   2 )
             self.assertEqual(ids[-1]			,   lowercase_ )
             self.assertEqual(len(lowercase_ )			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Dict )					->						Optional[int]:
             """simple docstring"""




             self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__'] )			,   [5_0004, 5_0001] )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=tempfile.mkdtemp()
             _lowerCamelCase       :		Dict		=self.tokenizer.fairseq_tokens_to_ids
             self.tokenizer.save_pretrained(lowercase_ )
             _lowerCamelCase       :		Any		=PLBartTokenizer.from_pretrained(lowercase_ )
             self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,   lowercase_ )






      @require_torch
      def        lowerCamelCase       (					self						:     Optional[Any] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=self.tokenizer(self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   return_tensors='pt' )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
             self.assertEqual(batch.input_ids[1][-2:].tolist()			,   [2, PYTHON_CODE] )
             self.assertEqual(batch.decoder_input_ids[1][0]			,   lowercase_ )
             self.assertEqual(batch.decoder_input_ids[1][-1]			,   2 )
             self.assertEqual(batch.labels[1][-2:].tolist()			,   [2, EN_CODE] )






      @require_torch
      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		int		=self.tokenizer(
                 self.src_text			,   text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=len(self.expected_src_tokens )			,   return_tensors='pt'			,   )
             _lowerCamelCase       :		List[Any]		=shift_tokens_right(batch['labels']			,   self.tokenizer.pad_token_id )

             self.assertIsInstance(lowercase_			,   lowercase_ )

             self.assertEqual((2, 26)			,   batch.input_ids.shape )
             self.assertEqual((2, 26)			,   batch.attention_mask.shape )
             _lowerCamelCase       :		List[Any]		=batch.input_ids.tolist()[0]
             self.assertListEqual(self.expected_src_tokens			,   lowercase_ )
             self.assertEqual(2			,   batch.decoder_input_ids[0, -1] )  # EOS
             # Test that special tokens are reset
             self.assertEqual(self.tokenizer.prefix_tokens			,   [] )
             self.assertEqual(self.tokenizer.suffix_tokens			,   [self.tokenizer.eos_token_id, PYTHON_CODE] )






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=self.tokenizer(self.src_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=3			,   return_tensors='pt' )
             _lowerCamelCase       :		Dict		=self.tokenizer(
                 text_target=self.tgt_text			,   padding=lowercase_			,   truncation=lowercase_			,   max_length=10			,   return_tensors='pt' )
             _lowerCamelCase       :		List[str]		=targets['input_ids']
             _lowerCamelCase       :		Optional[Any]		=shift_tokens_right(lowercase_			,   self.tokenizer.pad_token_id )

             self.assertEqual(batch.input_ids.shape[1]			,   3 )
             self.assertEqual(batch.decoder_input_ids.shape[1]			,   10 )






      @require_torch
      def        lowerCamelCase       (					self						:     Any )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Any		=self.tokenizer._build_translation_inputs(
                 'A test'			,   return_tensors='pt'			,   src_lang='en_XX'			,   tgt_lang='java' )

             self.assertEqual(
                 nested_simplify(lowercase_ )			,   {
                     # A, test, EOS, en_XX
                     'input_ids': [[150, 242, 2, 5_0003]],
                     'attention_mask': [[1, 1, 1, 1]],
                     # java
                     'forced_bos_token_id': 5_0001,
                 }			,   )


