'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCAmelCase     							=get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__UpperCAmelCase     							=2_5_6_0_4_7
__UpperCAmelCase     							=2_5_6_1_4_5




@require_sentencepiece
@require_tokenizers
class     a__      (   UpperCAmelCase__							,   unittest.TestCase ):
     lowerCamelCase  :     Union[str, Any]							=NllbTokenizer
     lowerCamelCase  :     Optional[int]							=NllbTokenizerFast
     lowerCamelCase  :     str							=True
     lowerCamelCase  :     str							=True
     lowerCamelCase  :     Tuple							={}






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[Any]    ):





         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __lowerCamelCase		=  NllbTokenizer(a  ,				keep_accents=a    )
         tokenizer.save_pretrained(self.tmpdirname    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Any    ):





         """simple docstring"""



         __lowerCamelCase		=  NllbTokenizer(a  ,				keep_accents=a    )

         __lowerCamelCase		=  tokenizer.tokenize('''This is a test'''    )
         self.assertListEqual(a  ,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']    )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(a    )  ,				[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]  ,				)

         __lowerCamelCase		=  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''    )
         self.assertListEqual(
             a  ,				[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ]  ,				)
         __lowerCamelCase		=  tokenizer.convert_tokens_to_ids(a    )
         self.assertListEqual(
             a  ,				[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]  ,				)

         __lowerCamelCase		=  tokenizer.convert_ids_to_tokens(a    )
         self.assertListEqual(
             a  ,				[
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ]  ,				)






     def    SCREAMING_SNAKE_CASE__   ( self		:		Optional[Any]    ):





         """simple docstring"""



         __lowerCamelCase		=  (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
                 __lowerCamelCase		=  self.rust_tokenizer_class.from_pretrained(a  ,				**a    )
                 __lowerCamelCase		=  self.tokenizer_class.from_pretrained(a  ,				**a    )

                 __lowerCamelCase		=  tempfile.mkdtemp()

                 __lowerCamelCase		=  tokenizer_r.save_pretrained(a    )
                 __lowerCamelCase		=  tokenizer_p.save_pretrained(a    )

                 # Checks it save with the same files + the tokenizer.json file for the fast one
                 self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files    )    )
                 __lowerCamelCase		=  tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f    )
                 self.assertSequenceEqual(a  ,				a    )

                 # Checks everything loads correctly in the same way
                 __lowerCamelCase		=  tokenizer_r.from_pretrained(a    )
                 __lowerCamelCase		=  tokenizer_p.from_pretrained(a    )

                 # Check special tokens are set accordingly on Rust and Python
                 for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(a  ,				a    )    )

                 shutil.rmtree(a    )

                 # Save tokenizer rust, legacy_format=True
                 __lowerCamelCase		=  tempfile.mkdtemp()

                 __lowerCamelCase		=  tokenizer_r.save_pretrained(a  ,				legacy_format=a    )
                 __lowerCamelCase		=  tokenizer_p.save_pretrained(a    )

                 # Checks it save with the same files
                 self.assertSequenceEqual(a  ,				a    )

                 # Checks everything loads correctly in the same way
                 __lowerCamelCase		=  tokenizer_r.from_pretrained(a    )
                 __lowerCamelCase		=  tokenizer_p.from_pretrained(a    )

                 # Check special tokens are set accordingly on Rust and Python
                 for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(a  ,				a    )    )

                 shutil.rmtree(a    )

                 # Save tokenizer rust, legacy_format=False
                 __lowerCamelCase		=  tempfile.mkdtemp()

                 __lowerCamelCase		=  tokenizer_r.save_pretrained(a  ,				legacy_format=a    )
                 __lowerCamelCase		=  tokenizer_p.save_pretrained(a    )

                 # Checks it saved the tokenizer.json file
                 self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files    )    )

                 # Checks everything loads correctly in the same way
                 __lowerCamelCase		=  tokenizer_r.from_pretrained(a    )
                 __lowerCamelCase		=  tokenizer_p.from_pretrained(a    )

                 # Check special tokens are set accordingly on Rust and Python
                 for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(a  ,				a    )    )

                 shutil.rmtree(a    )






     @require_torch
     def    SCREAMING_SNAKE_CASE__   ( self		:		List[str]    ):





         """simple docstring"""



         if not self.test_seqaseq:
             return

         __lowerCamelCase		=  self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
                 # Longer text that will definitely require truncation.
                 __lowerCamelCase		=  [
                     ''' UN Chief Says There Is No Military Solution in Syria''',
                     ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                     ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                     ''' will only worsen the violence and misery for millions of people.''',
                 ]
                 __lowerCamelCase		=  [
                     '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                     '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                     ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                     ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                 ]
                 try:
                     __lowerCamelCase		=  tokenizer.prepare_seqaseq_batch(
                         src_texts=a  ,				tgt_texts=a  ,				max_length=3  ,				max_target_length=10  ,				return_tensors='''pt'''  ,				src_lang='''eng_Latn'''  ,				tgt_lang='''ron_Latn'''  ,				)
                 except NotImplementedError:
                     return
                 self.assertEqual(batch.input_ids.shape[1]  ,				3    )
                 self.assertEqual(batch.labels.shape[1]  ,				10    )
                 # max_target_length will default to max_length if not specified
                 __lowerCamelCase		=  tokenizer.prepare_seqaseq_batch(
                     a  ,				tgt_texts=a  ,				max_length=3  ,				return_tensors='''pt'''    )
                 self.assertEqual(batch.input_ids.shape[1]  ,				3    )
                 self.assertEqual(batch.labels.shape[1]  ,				3    )

                 __lowerCamelCase		=  tokenizer.prepare_seqaseq_batch(
                     src_texts=a  ,				max_length=3  ,				max_target_length=10  ,				return_tensors='''pt'''    )
                 self.assertEqual(batch_encoder_only.input_ids.shape[1]  ,				3    )
                 self.assertEqual(batch_encoder_only.attention_mask.shape[1]  ,				3    )
                 self.assertNotIn('''decoder_input_ids'''  ,				a    )






     @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''    )
     def    SCREAMING_SNAKE_CASE__   ( self		:		Optional[int]    ):





         """simple docstring"""



         pass






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[Any]    ):





         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
                 __lowerCamelCase		=  [AddedToken('''<special>'''  ,				lstrip=a    )]

                 __lowerCamelCase		=  self.rust_tokenizer_class.from_pretrained(
                     a  ,				additional_special_tokens=a  ,				**a    )
                 __lowerCamelCase		=  tokenizer_r.encode('''Hey this is a <special> token'''    )

                 __lowerCamelCase		=  tokenizer_r.encode('''<special>'''  ,				add_special_tokens=a    )[0]

                 self.assertTrue(special_token_id in r_output    )

                 if self.test_slow_tokenizer:
                     __lowerCamelCase		=  self.rust_tokenizer_class.from_pretrained(
                         a  ,				additional_special_tokens=a  ,				**a  ,				)
                     __lowerCamelCase		=  self.tokenizer_class.from_pretrained(
                         a  ,				additional_special_tokens=a  ,				**a    )

                     __lowerCamelCase		=  tokenizer_p.encode('''Hey this is a <special> token'''    )

                     __lowerCamelCase		=  tokenizer_cr.encode('''Hey this is a <special> token'''    )

                     self.assertEqual(a  ,				a    )
                     self.assertEqual(a  ,				a    )
                     self.assertTrue(special_token_id in p_output    )
                     self.assertTrue(special_token_id in cr_output    )




@require_torch
@require_sentencepiece
@require_tokenizers
class     a__      (   unittest.TestCase ):
     lowerCamelCase  :     int							="facebook/nllb-200-distilled-600M"
     lowerCamelCase  :     Optional[int]							=[
         " UN Chief Says There Is No Military Solution in Syria",
         " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
     ]
     lowerCamelCase  :     Tuple							=[
         "Şeful ONU declară că nu există o soluţie militară în Siria",
         "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
         " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
         " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
     ]
     lowerCamelCase  :     Union[str, Any]							=[
         2_5_6_0_4_7,
         1_6_2_9_7,
         1_3_4_4_0_8,
         8_1_6_5,
         2_4_8_0_6_6,
         1_4_7_3_4,
         9_5_0,
         1_1_3_5,
         1_0_5_7_2_1,
         3_5_7_3,
         8_3,
         2_7_3_5_2,
         1_0_8,
         4_9_4_8_6,
         2,
     ]






     @classmethod
     def    SCREAMING_SNAKE_CASE__   ( cls		:		Tuple    ):





         """simple docstring"""



         __lowerCamelCase		=  NllbTokenizer.from_pretrained(
             cls.checkpoint_name  ,				src_lang='''eng_Latn'''  ,				tgt_lang='''ron_Latn'''    )
         __lowerCamelCase		=  1
         return cls






     def    SCREAMING_SNAKE_CASE__   ( self		:		str    ):





         """simple docstring"""



         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']  ,				25_60_01    )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']  ,				25_60_02    )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']  ,				25_60_57    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		int    ):





         """simple docstring"""



         __lowerCamelCase		=  self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens  ,				a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Union[str, Any]    ):





         """simple docstring"""



         self.assertIn(a  ,				self.tokenizer.all_special_ids    )
         # fmt: off
         __lowerCamelCase		=  [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
         # fmt: on

         __lowerCamelCase		=  self.tokenizer.decode(a  ,				skip_special_tokens=a    )
         __lowerCamelCase		=  self.tokenizer.decode(generated_ids[1:]  ,				skip_special_tokens=a    )
         self.assertEqual(a  ,				a    )
         self.assertNotIn(self.tokenizer.eos_token  ,				a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Union[str, Any]    ):





         """simple docstring"""



         __lowerCamelCase		=  ['''this is gunna be a long sentence ''' * 20]
         assert isinstance(src_text[0]  ,				a    )
         __lowerCamelCase		=  10
         __lowerCamelCase		=  self.tokenizer(a  ,				max_length=a  ,				truncation=a    ).input_ids[0]
         self.assertEqual(ids[-1]  ,				2    )
         self.assertEqual(ids[0]  ,				a    )
         self.assertEqual(len(a    )  ,				a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Optional[int]    ):





         """simple docstring"""



         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']    )  ,				[25_62_03, 3]    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		int    ):





         """simple docstring"""



         __lowerCamelCase		=  tempfile.mkdtemp()
         __lowerCamelCase		=  self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(a    )
         __lowerCamelCase		=  NllbTokenizer.from_pretrained(a    )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids  ,				a    )






     @require_torch
     def    SCREAMING_SNAKE_CASE__   ( self		:		List[Any]    ):





         """simple docstring"""



         __lowerCamelCase		=  self.tokenizer(
             self.src_text  ,				text_target=self.tgt_text  ,				padding=a  ,				truncation=a  ,				max_length=len(self.expected_src_tokens    )  ,				return_tensors='''pt'''  ,				)
         __lowerCamelCase		=  shift_tokens_right(
             batch['''labels''']  ,				self.tokenizer.pad_token_id  ,				self.tokenizer.lang_code_to_id['''ron_Latn''']    )

         self.assertIsInstance(a  ,				a    )

         self.assertEqual((2, 15)  ,				batch.input_ids.shape    )
         self.assertEqual((2, 15)  ,				batch.attention_mask.shape    )
         __lowerCamelCase		=  batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens  ,				a    )
         self.assertEqual(a  ,				batch.decoder_input_ids[0, 0]    )  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens  ,				[EN_CODE]    )
         self.assertEqual(self.tokenizer.suffix_tokens  ,				[self.tokenizer.eos_token_id]    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Optional[Any]    ):





         """simple docstring"""



         __lowerCamelCase		=  self.tokenizer(self.src_text  ,				padding=a  ,				truncation=a  ,				max_length=3  ,				return_tensors='''pt'''    )
         __lowerCamelCase		=  self.tokenizer(
             text_target=self.tgt_text  ,				padding=a  ,				truncation=a  ,				max_length=10  ,				return_tensors='''pt'''    )
         __lowerCamelCase		=  targets['''input_ids''']
         __lowerCamelCase		=  shift_tokens_right(
             a  ,				self.tokenizer.pad_token_id  ,				decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]  ,				)

         self.assertEqual(batch.input_ids.shape[1]  ,				3    )
         self.assertEqual(batch.decoder_input_ids.shape[1]  ,				10    )






     @require_torch
     def    SCREAMING_SNAKE_CASE__   ( self		:		Any    ):





         """simple docstring"""



         __lowerCamelCase		=  self.tokenizer._build_translation_inputs(
             '''A test'''  ,				return_tensors='''pt'''  ,				src_lang='''eng_Latn'''  ,				tgt_lang='''fra_Latn'''    )

         self.assertEqual(
             nested_simplify(a    )  ,				{
                 # A, test, EOS, en_XX
                 '''input_ids''': [[25_60_47, 70, 73_56, 2]],
                 '''attention_mask''': [[1, 1, 1, 1]],
                 # ar_AR
                 '''forced_bos_token_id''': 25_60_57,
             }  ,				)






     @require_torch
     def    SCREAMING_SNAKE_CASE__   ( self		:		Any    ):





         """simple docstring"""



         __lowerCamelCase		=  True
         __lowerCamelCase		=  self.tokenizer(
             '''UN Chief says there is no military solution in Syria'''  ,				src_lang='''eng_Latn'''  ,				tgt_lang='''fra_Latn'''    )
         self.assertEqual(
             inputs.input_ids  ,				[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]    )

         __lowerCamelCase		=  False
         __lowerCamelCase		=  self.tokenizer(
             '''UN Chief says there is no military solution in Syria'''  ,				src_lang='''eng_Latn'''  ,				tgt_lang='''fra_Latn'''    )
         self.assertEqual(
             inputs.input_ids  ,				[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]    )

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


a_    					=       {
    'configuration_efficientformer': [
        'EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'EfficientFormerConfig',
    ]
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   a_    					=       ['EfficientFormerImageProcessor']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   a_    					=       [
       'EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
       'EfficientFormerForImageClassification',
       'EfficientFormerForImageClassificationWithTeacher',
       'EfficientFormerModel',
       'EfficientFormerPreTrainedModel',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   a_    					=       [
       'TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
       'TFEfficientFormerForImageClassification',
       'TFEfficientFormerForImageClassificationWithTeacher',
       'TFEfficientFormerModel',
       'TFEfficientFormerPreTrainedModel',
   ]

if TYPE_CHECKING:
   from .configuration_efficientformer import EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, EfficientFormerConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_efficientformer import EfficientFormerImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_efficientformer import (
          EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          EfficientFormerForImageClassification,
          EfficientFormerForImageClassificationWithTeacher,
          EfficientFormerModel,
          EfficientFormerPreTrainedModel,
      )
   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_efficientformer import (
          TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFEfficientFormerForImageClassification,
          TFEfficientFormerForImageClassificationWithTeacher,
          TFEfficientFormerModel,
          TFEfficientFormerPreTrainedModel,
      )

else:
   import sys

   a_    					=       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



