






'''simple docstring'''





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase		:		List[str]            = get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_UpperCamelCase		:		Tuple            = 256_047
_UpperCamelCase		:		List[str]            = 256_145


@require_sentencepiece
@require_tokenizers
class       snake_case__	( a_			,							unittest.TestCase):
					a_								= NllbTokenizer
					a_								= NllbTokenizerFast
					a_								= True
					a_								= True
					a_								= {}




					def 						A	(   self   :						Dict	)  ->				Optional[Any]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase_	:    Optional[int]	  =    NllbTokenizer(_a							,      keep_accents=_a	)
								tokenizer.save_pretrained(self.tmpdirname	)




					def 						A	(   self   :						Dict	)  ->				List[Any]:
								UpperCAmelCase_	:    Tuple	  =    NllbTokenizer(_a							,      keep_accents=_a	)

								UpperCAmelCase_	:    List[Any]	  =    tokenizer.tokenize('''This is a test'''	)
								self.assertListEqual(_a							,      ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']	)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_a	)							,      [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]							,      )

								UpperCAmelCase_	:    List[str]	  =    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''	)
								self.assertListEqual(
								    _a							,      [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ]							,      )
								UpperCAmelCase_	:    Union[str, Any]	  =    tokenizer.convert_tokens_to_ids(_a	)
								self.assertListEqual(
								    _a							,      [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]							,      )

								UpperCAmelCase_	:    List[Any]	  =    tokenizer.convert_ids_to_tokens(_a	)
								self.assertListEqual(
								    _a							,      [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''<unk>''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''<unk>''',
								        '''.''',
								    ]							,      )




					def 						A	(   self   :						Optional[Any]	)  ->				Tuple:
								UpperCAmelCase_	:    Union[str, Any]	  =    (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
														UpperCAmelCase_	:    str	  =    self.rust_tokenizer_class.from_pretrained(_a							,      **_a	)
														UpperCAmelCase_	:    Optional[Any]	  =    self.tokenizer_class.from_pretrained(_a							,      **_a	)

														UpperCAmelCase_	:    Any	  =    tempfile.mkdtemp()

														UpperCAmelCase_	:    Dict	  =    tokenizer_r.save_pretrained(_a	)
														UpperCAmelCase_	:    int	  =    tokenizer_p.save_pretrained(_a	)

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files	)	)
														UpperCAmelCase_	:    Tuple	  =    tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f	)
														self.assertSequenceEqual(_a							,      _a	)

														# Checks everything loads correctly in the same way
														UpperCAmelCase_	:    List[Any]	  =    tokenizer_r.from_pretrained(_a	)
														UpperCAmelCase_	:    Tuple	  =    tokenizer_p.from_pretrained(_a	)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(_a							,      _a	)	)

														shutil.rmtree(_a	)

														# Save tokenizer rust, legacy_format=True
														UpperCAmelCase_	:    Any	  =    tempfile.mkdtemp()

														UpperCAmelCase_	:    str	  =    tokenizer_r.save_pretrained(_a							,      legacy_format=_a	)
														UpperCAmelCase_	:    Optional[Any]	  =    tokenizer_p.save_pretrained(_a	)

														# Checks it save with the same files
														self.assertSequenceEqual(_a							,      _a	)

														# Checks everything loads correctly in the same way
														UpperCAmelCase_	:    List[Any]	  =    tokenizer_r.from_pretrained(_a	)
														UpperCAmelCase_	:    Dict	  =    tokenizer_p.from_pretrained(_a	)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(_a							,      _a	)	)

														shutil.rmtree(_a	)

														# Save tokenizer rust, legacy_format=False
														UpperCAmelCase_	:    Union[str, Any]	  =    tempfile.mkdtemp()

														UpperCAmelCase_	:    Dict	  =    tokenizer_r.save_pretrained(_a							,      legacy_format=_a	)
														UpperCAmelCase_	:    int	  =    tokenizer_p.save_pretrained(_a	)

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files	)	)

														# Checks everything loads correctly in the same way
														UpperCAmelCase_	:    str	  =    tokenizer_r.from_pretrained(_a	)
														UpperCAmelCase_	:    Dict	  =    tokenizer_p.from_pretrained(_a	)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(_a							,      _a	)	)

														shutil.rmtree(_a	)




					@require_torch
					def 						A	(   self   :						Optional[Any]	)  ->				Tuple:
								if not self.test_seqaseq:
											return

								UpperCAmelCase_	:    List[str]	  =    self.get_tokenizers()
								for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"	):
														# Longer text that will definitely require truncation.
														UpperCAmelCase_	:    Union[str, Any]	  =    [
														    " UN Chief Says There Is No Military Solution in Syria",
														    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
														    " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
														    " will only worsen the violence and misery for millions of people.",
														]
														UpperCAmelCase_	:    Dict	  =    [
														    "Şeful ONU declară că nu există o soluţie militară în Siria",
														    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
														    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
														    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
														]
														try:
																	UpperCAmelCase_	:    List[Any]	  =    tokenizer.prepare_seqaseq_batch(
																	    src_texts=_a							,      tgt_texts=_a							,      max_length=3							,      max_target_length=10							,      return_tensors='''pt'''							,      src_lang='''eng_Latn'''							,      tgt_lang='''ron_Latn'''							,      )
														except NotImplementedError:
																	return
														self.assertEqual(batch.input_ids.shape[1]							,      3	)
														self.assertEqual(batch.labels.shape[1]							,      10	)
														# max_target_length will default to max_length if not specified
														UpperCAmelCase_	:    Optional[Any]	  =    tokenizer.prepare_seqaseq_batch(
														    _a							,      tgt_texts=_a							,      max_length=3							,      return_tensors='''pt'''	)
														self.assertEqual(batch.input_ids.shape[1]							,      3	)
														self.assertEqual(batch.labels.shape[1]							,      3	)

														UpperCAmelCase_	:    int	  =    tokenizer.prepare_seqaseq_batch(
														    src_texts=_a							,      max_length=3							,      max_target_length=10							,      return_tensors='''pt'''	)
														self.assertEqual(batch_encoder_only.input_ids.shape[1]							,      3	)
														self.assertEqual(batch_encoder_only.attention_mask.shape[1]							,      3	)
														self.assertNotIn('''decoder_input_ids'''							,      _a	)




					@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''	)
					def 						A	(   self   :						Union[str, Any]	)  ->				str:
								pass





					def 						A	(   self   :						List[Any]	)  ->				Optional[int]:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
														UpperCAmelCase_	:    Any	  =    [AddedToken('''<special>'''							,      lstrip=_a	)]

														UpperCAmelCase_	:    Any	  =    self.rust_tokenizer_class.from_pretrained(
														    _a							,      additional_special_tokens=_a							,      **_a	)
														UpperCAmelCase_	:    List[Any]	  =    tokenizer_r.encode('''Hey this is a <special> token'''	)

														UpperCAmelCase_	:    Dict	  =    tokenizer_r.encode('''<special>'''							,      add_special_tokens=_a	)[0]

														self.assertTrue(special_token_id in r_output	)

														if self.test_slow_tokenizer:
																	UpperCAmelCase_	:    Optional[Any]	  =    self.rust_tokenizer_class.from_pretrained(
																	    _a							,      additional_special_tokens=_a							,      **_a							,      )
																	UpperCAmelCase_	:    Union[str, Any]	  =    self.tokenizer_class.from_pretrained(
																	    _a							,      additional_special_tokens=_a							,      **_a	)

																	UpperCAmelCase_	:    List[Any]	  =    tokenizer_p.encode('''Hey this is a <special> token'''	)

																	UpperCAmelCase_	:    List[str]	  =    tokenizer_cr.encode('''Hey this is a <special> token'''	)

																	self.assertEqual(_a							,      _a	)
																	self.assertEqual(_a							,      _a	)
																	self.assertTrue(special_token_id in p_output	)
																	self.assertTrue(special_token_id in cr_output	)






@require_torch
@require_sentencepiece
@require_tokenizers
class       snake_case__	( unittest.TestCase):
					a_								= "facebook/nllb-200-distilled-600M"
					a_								= [
					    " UN Chief Says There Is No Military Solution in Syria",
					    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
					]
					a_								= [
					    "Şeful ONU declară că nu există o soluţie militară în Siria",
					    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
					    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
					    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
					]
					a_								= [
					    256047,
					    16297,
					    134408,
					    8165,
					    248066,
					    14734,
					    950,
					    1135,
					    105721,
					    3573,
					    83,
					    27352,
					    108,
					    49486,
					    2,
					]




					@classmethod
					def 						A	(   cls   :						Dict	)  ->				Optional[int]:
								UpperCAmelCase_	:    NllbTokenizer	  =    NllbTokenizer.from_pretrained(
								    cls.checkpoint_name							,      src_lang='''eng_Latn'''							,      tgt_lang='''ron_Latn'''	)
								UpperCAmelCase_	:    Optional[int]	  =    1
								return cls




					def 						A	(   self   :						int	)  ->				List[Any]:
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']							,      25_60_01	)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']							,      25_60_02	)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']							,      25_60_57	)




					def 						A	(   self   :						str	)  ->				Any:
								UpperCAmelCase_	:    int	  =    self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
								self.assertListEqual(self.expected_src_tokens							,      _a	)




					def 						A	(   self   :						Any	)  ->				List[str]:
								self.assertIn(_a							,      self.tokenizer.all_special_ids	)
								# fmt: off
								UpperCAmelCase_	:    int	  =    [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
								# fmt: on

								UpperCAmelCase_	:    Union[str, Any]	  =    self.tokenizer.decode(_a							,      skip_special_tokens=_a	)
								UpperCAmelCase_	:    str	  =    self.tokenizer.decode(generated_ids[1:]							,      skip_special_tokens=_a	)
								self.assertEqual(_a							,      _a	)
								self.assertNotIn(self.tokenizer.eos_token							,      _a	)




					def 						A	(   self   :						str	)  ->				Optional[int]:
								UpperCAmelCase_	:    Tuple	  =    ["this is gunna be a long sentence " * 20]
								assert isinstance(src_text[0]							,      _a	)
								UpperCAmelCase_	:    Optional[Any]	  =    10
								UpperCAmelCase_	:    List[str]	  =    self.tokenizer(_a							,      max_length=_a							,      truncation=_a	).input_ids[0]
								self.assertEqual(ids[-1]							,      2	)
								self.assertEqual(ids[0]							,      _a	)
								self.assertEqual(len(_a	)							,      _a	)




					def 						A	(   self   :						List[str]	)  ->				Any:
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']	)							,      [25_62_03, 3]	)




					def 						A	(   self   :						List[str]	)  ->				Any:
								UpperCAmelCase_	:    Optional[int]	  =    tempfile.mkdtemp()
								UpperCAmelCase_	:    Optional[int]	  =    self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(_a	)
								UpperCAmelCase_	:    Tuple	  =    NllbTokenizer.from_pretrained(_a	)
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,      _a	)




					@require_torch
					def 						A	(   self   :						str	)  ->				Optional[int]:
								UpperCAmelCase_	:    Dict	  =    self.tokenizer(
								    self.src_text							,      text_target=self.tgt_text							,      padding=_a							,      truncation=_a							,      max_length=len(self.expected_src_tokens	)							,      return_tensors='''pt'''							,      )
								UpperCAmelCase_	:    str	  =    shift_tokens_right(
								    batch['''labels''']							,      self.tokenizer.pad_token_id							,      self.tokenizer.lang_code_to_id['''ron_Latn''']	)

								self.assertIsInstance(_a							,      _a	)

								self.assertEqual((2, 15)							,      batch.input_ids.shape	)
								self.assertEqual((2, 15)							,      batch.attention_mask.shape	)
								UpperCAmelCase_	:    Any	  =    batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens							,      _a	)
								self.assertEqual(_a							,      batch.decoder_input_ids[0, 0]	)  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens							,      [EN_CODE]	)
								self.assertEqual(self.tokenizer.suffix_tokens							,      [self.tokenizer.eos_token_id]	)




					def 						A	(   self   :						Optional[int]	)  ->				Dict:
								UpperCAmelCase_	:    Optional[Any]	  =    self.tokenizer(self.src_text							,      padding=_a							,      truncation=_a							,      max_length=3							,      return_tensors='''pt'''	)
								UpperCAmelCase_	:    Optional[int]	  =    self.tokenizer(
								    text_target=self.tgt_text							,      padding=_a							,      truncation=_a							,      max_length=10							,      return_tensors='''pt'''	)
								UpperCAmelCase_	:    Optional[int]	  =    targets["input_ids"]
								UpperCAmelCase_	:    Union[str, Any]	  =    shift_tokens_right(
								    _a							,      self.tokenizer.pad_token_id							,      decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]							,      )

								self.assertEqual(batch.input_ids.shape[1]							,      3	)
								self.assertEqual(batch.decoder_input_ids.shape[1]							,      10	)




					@require_torch
					def 						A	(   self   :						Dict	)  ->				int:
								UpperCAmelCase_	:    Dict	  =    self.tokenizer._build_translation_inputs(
								    '''A test'''							,      return_tensors='''pt'''							,      src_lang='''eng_Latn'''							,      tgt_lang='''fra_Latn'''	)

								self.assertEqual(
								    nested_simplify(_a	)							,      {
								        # A, test, EOS, en_XX
								        '''input_ids''': [[25_60_47, 70, 73_56, 2]],
								        '''attention_mask''': [[1, 1, 1, 1]],
								        # ar_AR
								        '''forced_bos_token_id''': 25_60_57,
								    }							,      )





					@require_torch
					def 						A	(   self   :						int	)  ->				Optional[int]:
								UpperCAmelCase_	:    str	  =    True
								UpperCAmelCase_	:    Any	  =    self.tokenizer(
								    '''UN Chief says there is no military solution in Syria'''							,      src_lang='''eng_Latn'''							,      tgt_lang='''fra_Latn'''	)
								self.assertEqual(
								    inputs.input_ids							,      [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]	)

								UpperCAmelCase_	:    Any	  =    False
								UpperCAmelCase_	:    Dict	  =    self.tokenizer(
								    '''UN Chief says there is no military solution in Syria'''							,      src_lang='''eng_Latn'''							,      tgt_lang='''fra_Latn'''	)
								self.assertEqual(
								    inputs.input_ids							,      [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]	)




"""simple docstring"""




from __future__ import annotations

from functools import lru_cache
from math import ceil

_A			:       Optional[Any]   		=       1_00

_A			:       Optional[int]   		=       set(range(3, NUM_PRIMES, 2))
primes.add(2)
_A			:       int

for prime in range(3, ceil(NUM_PRIMES**0.5), 2):
   if prime not in primes:
      continue
   primes.difference_update(set(range(prime * prime, NUM_PRIMES, prime)))


@lru_cache(maxsize=100  )
def   __magic_name__	(      __snake_case				:       int  ) ->				set[int]:
       if number_to_partition < 0:
              return set()
       elif number_to_partition == 0:
              return {1}

       lowercase    :				set[int]       		=							set()
       lowercase    :				int
       lowercase    :				int

       for prime in primes:
              if prime > number_to_partition:
                     continue
              for sub in partition(number_to_partition - prime  ):
                     ret.add(sub * prime  )

       return ret


def   __magic_name__	(      __snake_case				:       int = 5000  ) ->				int | None:
       for number_to_partition in range(1     , __snake_case  ):
              if len(partition(__snake_case  )  ) > number_unique_partitions:
                     return number_to_partition
       return None


if __name__ == "__main__":
   print(F"{solution() = }")


