





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


a__:      Any									=		get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
       from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

a__:      str									=		256_047
a__:      Union[str, Any]									=		256_145




@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__ (						UpperCamelCase__				,					unittest.TestCase      ):
 __SCREAMING_SNAKE_CASE							       =	NllbTokenizer
 __SCREAMING_SNAKE_CASE							       =	NllbTokenizerFast
 __SCREAMING_SNAKE_CASE							       =	True
 __SCREAMING_SNAKE_CASE							       =	True
 __SCREAMING_SNAKE_CASE							       =	{}



 def 				UpperCamelCase			(							self ):
        super().setUp()

        # We have a SentencePiece fixture for testing
        A__											=						NllbTokenizer(__lowerCamelCase,keep_accents=__lowerCamelCase )
        tokenizer.save_pretrained(self.tmpdirname )



 def 				UpperCamelCase			(							self ):
        A__											=						NllbTokenizer(__lowerCamelCase,keep_accents=__lowerCamelCase )

        A__											=						tokenizer.tokenize('''This is a test''' )
        self.assertListEqual(__lowerCamelCase,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__lowerCamelCase ),[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]],)

        A__											=						tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
        self.assertListEqual(
            __lowerCamelCase,[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ],)
        A__											=						tokenizer.convert_tokens_to_ids(__lowerCamelCase )
        self.assertListEqual(
            __lowerCamelCase,[
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ],)

        A__											=						tokenizer.convert_ids_to_tokens(__lowerCamelCase )
        self.assertListEqual(
            __lowerCamelCase,[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ],)



 def 				UpperCamelCase			(							self ):
        A__											=						(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
                      A__											=						self.rust_tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )
                      A__											=						self.tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase )

                      A__											=						tempfile.mkdtemp()

                      A__											=						tokenizer_r.save_pretrained(__lowerCamelCase )
                      A__											=						tokenizer_p.save_pretrained(__lowerCamelCase )

                      # Checks it save with the same files + the tokenizer.json file for the fast one
                      self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )
                      A__											=						tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f )
                      self.assertSequenceEqual(__lowerCamelCase,__lowerCamelCase )

                      # Checks everything loads correctly in the same way
                      A__											=						tokenizer_r.from_pretrained(__lowerCamelCase )
                      A__											=						tokenizer_p.from_pretrained(__lowerCamelCase )

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(__lowerCamelCase,__lowerCamelCase ) )

                      shutil.rmtree(__lowerCamelCase )

                      # Save tokenizer rust, legacy_format=True
                      A__											=						tempfile.mkdtemp()

                      A__											=						tokenizer_r.save_pretrained(__lowerCamelCase,legacy_format=__lowerCamelCase )
                      A__											=						tokenizer_p.save_pretrained(__lowerCamelCase )

                      # Checks it save with the same files
                      self.assertSequenceEqual(__lowerCamelCase,__lowerCamelCase )

                      # Checks everything loads correctly in the same way
                      A__											=						tokenizer_r.from_pretrained(__lowerCamelCase )
                      A__											=						tokenizer_p.from_pretrained(__lowerCamelCase )

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(__lowerCamelCase,__lowerCamelCase ) )

                      shutil.rmtree(__lowerCamelCase )

                      # Save tokenizer rust, legacy_format=False
                      A__											=						tempfile.mkdtemp()

                      A__											=						tokenizer_r.save_pretrained(__lowerCamelCase,legacy_format=__lowerCamelCase )
                      A__											=						tokenizer_p.save_pretrained(__lowerCamelCase )

                      # Checks it saved the tokenizer.json file
                      self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files ) )

                      # Checks everything loads correctly in the same way
                      A__											=						tokenizer_r.from_pretrained(__lowerCamelCase )
                      A__											=						tokenizer_p.from_pretrained(__lowerCamelCase )

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(__lowerCamelCase,__lowerCamelCase ) )

                      shutil.rmtree(__lowerCamelCase )



 @require_torch
 def 				UpperCamelCase			(							self ):
        if not self.test_seqaseq:
               return

        A__											=						self.get_tokenizers()
        for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}" ):
                      # Longer text that will definitely require truncation.
                      A__											=						[
                          ''' UN Chief Says There Is No Military Solution in Syria''',
                          ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                          ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                          ''' will only worsen the violence and misery for millions of people.''',
                      ]
                      A__											=						[
                          '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                          '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                          ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                          ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                      ]
                      try:
                             A__											=						tokenizer.prepare_seqaseq_batch(
                                 src_texts=__lowerCamelCase,tgt_texts=__lowerCamelCase,max_length=3,max_target_length=10,return_tensors='''pt''',src_lang='''eng_Latn''',tgt_lang='''ron_Latn''',)
                      except NotImplementedError:
                             return
                      self.assertEqual(batch.input_ids.shape[1],3 )
                      self.assertEqual(batch.labels.shape[1],10 )
                      # max_target_length will default to max_length if not specified
                      A__											=						tokenizer.prepare_seqaseq_batch(
                          __lowerCamelCase,tgt_texts=__lowerCamelCase,max_length=3,return_tensors='''pt''' )
                      self.assertEqual(batch.input_ids.shape[1],3 )
                      self.assertEqual(batch.labels.shape[1],3 )

                      A__											=						tokenizer.prepare_seqaseq_batch(
                          src_texts=__lowerCamelCase,max_length=3,max_target_length=10,return_tensors='''pt''' )
                      self.assertEqual(batch_encoder_only.input_ids.shape[1],3 )
                      self.assertEqual(batch_encoder_only.attention_mask.shape[1],3 )
                      self.assertNotIn('''decoder_input_ids''',__lowerCamelCase )



 @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.''' )
 def 				UpperCamelCase			(							self ):
        pass




 def 				UpperCamelCase			(							self ):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
                      A__											=						[AddedToken('''<special>''',lstrip=__lowerCamelCase )]

                      A__											=						self.rust_tokenizer_class.from_pretrained(
                          __lowerCamelCase,additional_special_tokens=__lowerCamelCase,**__lowerCamelCase )
                      A__											=						tokenizer_r.encode('''Hey this is a <special> token''' )

                      A__											=						tokenizer_r.encode('''<special>''',add_special_tokens=__lowerCamelCase )[0]

                      self.assertTrue(special_token_id in r_output )

                      if self.test_slow_tokenizer:
                             A__											=						self.rust_tokenizer_class.from_pretrained(
                                 __lowerCamelCase,additional_special_tokens=__lowerCamelCase,**__lowerCamelCase,)
                             A__											=						self.tokenizer_class.from_pretrained(
                                 __lowerCamelCase,additional_special_tokens=__lowerCamelCase,**__lowerCamelCase )

                             A__											=						tokenizer_p.encode('''Hey this is a <special> token''' )

                             A__											=						tokenizer_cr.encode('''Hey this is a <special> token''' )

                             self.assertEqual(__lowerCamelCase,__lowerCamelCase )
                             self.assertEqual(__lowerCamelCase,__lowerCamelCase )
                             self.assertTrue(special_token_id in p_output )
                             self.assertTrue(special_token_id in cr_output )




@require_torch
@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__ (						unittest.TestCase      ):
 __SCREAMING_SNAKE_CASE							       =	'''facebook/nllb-200-distilled-600M'''
 __SCREAMING_SNAKE_CASE							       =	[
     ''' UN Chief Says There Is No Military Solution in Syria''',
     ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
 ]
 __SCREAMING_SNAKE_CASE							       =	[
     '''Şeful ONU declară că nu există o soluţie militară în Siria''',
     '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
     ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
     ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
 ]
 __SCREAMING_SNAKE_CASE							       =	[
     25_60_47,
     1_62_97,
     13_44_08,
     81_65,
     24_80_66,
     1_47_34,
     9_50,
     11_35,
     10_57_21,
     35_73,
     83,
     2_73_52,
     1_08,
     4_94_86,
     2,
 ]



 @classmethod
 def 				UpperCamelCase			(							cls ):
        A__											=						NllbTokenizer.from_pretrained(
            cls.checkpoint_name,src_lang='''eng_Latn''',tgt_lang='''ron_Latn''' )
        A__											=						1
        return cls



 def 				UpperCamelCase			(							self ):
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab'''],25_6001 )
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn'''],25_6002 )
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn'''],25_6057 )



 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
        self.assertListEqual(self.expected_src_tokens,__lowerCamelCase )



 def 				UpperCamelCase			(							self ):
        self.assertIn(__lowerCamelCase,self.tokenizer.all_special_ids )
        # fmt: off
        A__											=						[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
        # fmt: on

        A__											=						self.tokenizer.decode(__lowerCamelCase,skip_special_tokens=__lowerCamelCase )
        A__											=						self.tokenizer.decode(generated_ids[1:],skip_special_tokens=__lowerCamelCase )
        self.assertEqual(__lowerCamelCase,__lowerCamelCase )
        self.assertNotIn(self.tokenizer.eos_token,__lowerCamelCase )



 def 				UpperCamelCase			(							self ):
        A__											=						['''this is gunna be a long sentence ''' * 20]
        assert isinstance(src_text[0],__lowerCamelCase )
        A__											=						10
        A__											=						self.tokenizer(__lowerCamelCase,max_length=__lowerCamelCase,truncation=__lowerCamelCase ).input_ids[0]
        self.assertEqual(ids[-1],2 )
        self.assertEqual(ids[0],__lowerCamelCase )
        self.assertEqual(len(__lowerCamelCase ),__lowerCamelCase )



 def 				UpperCamelCase			(							self ):
        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR'''] ),[25_6203, 3] )



 def 				UpperCamelCase			(							self ):
        A__											=						tempfile.mkdtemp()
        A__											=						self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(__lowerCamelCase )
        A__											=						NllbTokenizer.from_pretrained(__lowerCamelCase )
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids,__lowerCamelCase )



 @require_torch
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer(
            self.src_text,text_target=self.tgt_text,padding=__lowerCamelCase,truncation=__lowerCamelCase,max_length=len(self.expected_src_tokens ),return_tensors='''pt''',)
        A__											=						shift_tokens_right(
            batch['''labels'''],self.tokenizer.pad_token_id,self.tokenizer.lang_code_to_id['''ron_Latn'''] )

        self.assertIsInstance(__lowerCamelCase,__lowerCamelCase )

        self.assertEqual((2, 15),batch.input_ids.shape )
        self.assertEqual((2, 15),batch.attention_mask.shape )
        A__											=						batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens,__lowerCamelCase )
        self.assertEqual(__lowerCamelCase,batch.decoder_input_ids[0, 0] )  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens,[EN_CODE] )
        self.assertEqual(self.tokenizer.suffix_tokens,[self.tokenizer.eos_token_id] )



 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer(self.src_text,padding=__lowerCamelCase,truncation=__lowerCamelCase,max_length=3,return_tensors='''pt''' )
        A__											=						self.tokenizer(
            text_target=self.tgt_text,padding=__lowerCamelCase,truncation=__lowerCamelCase,max_length=10,return_tensors='''pt''' )
        A__											=						targets['''input_ids''']
        A__											=						shift_tokens_right(
            __lowerCamelCase,self.tokenizer.pad_token_id,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang],)

        self.assertEqual(batch.input_ids.shape[1],3 )
        self.assertEqual(batch.decoder_input_ids.shape[1],10 )



 @require_torch
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer._build_translation_inputs(
            '''A test''',return_tensors='''pt''',src_lang='''eng_Latn''',tgt_lang='''fra_Latn''' )

        self.assertEqual(
            nested_simplify(__lowerCamelCase ),{
                # A, test, EOS, en_XX
                '''input_ids''': [[25_6047, 70, 7356, 2]],
                '''attention_mask''': [[1, 1, 1, 1]],
                # ar_AR
                '''forced_bos_token_id''': 25_6057,
            },)




 @require_torch
 def 				UpperCamelCase			(							self ):
        A__											=						True
        A__											=						self.tokenizer(
            '''UN Chief says there is no military solution in Syria''',src_lang='''eng_Latn''',tgt_lang='''fra_Latn''' )
        self.assertEqual(
            inputs.input_ids,[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047] )

        A__											=						False
        A__											=						self.tokenizer(
            '''UN Chief says there is no military solution in Syria''',src_lang='''eng_Latn''',tgt_lang='''fra_Latn''' )
        self.assertEqual(
            inputs.input_ids,[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2] )












class     SCREAMING_SNAKE_CASE__ :



 def __init__(							self,__lowerCamelCase ):
        A__											=						set_counts
        A__											=						max(__lowerCamelCase )
        A__											=						len(__lowerCamelCase )
        A__											=						[1] * num_sets
        A__											=						list(range(__lowerCamelCase ) )



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase ):
        A__											=						self.get_parent(__lowerCamelCase )
        A__											=						self.get_parent(__lowerCamelCase )

        if src_parent == dst_parent:
               return False

        if self.ranks[dst_parent] >= self.ranks[src_parent]:
               self.set_counts[dst_parent] += self.set_counts[src_parent]
               A__											=						0
               A__											=						dst_parent
               if self.ranks[dst_parent] == self.ranks[src_parent]:
                      self.ranks[dst_parent] += 1
               A__											=						self.set_counts[dst_parent]
        else:
               self.set_counts[src_parent] += self.set_counts[dst_parent]
               A__											=						0
               A__											=						src_parent
               A__											=						self.set_counts[src_parent]

        A__											=						max(self.max_set,__lowerCamelCase )
        return True




 def 				UpperCamelCase			(							self,__lowerCamelCase ):
        if self.parents[disj_set] == disj_set:
               return disj_set
        A__											=						self.get_parent(self.parents[disj_set] )
        return self.parents[disj_set]





