





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_     							= get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase_     							= 25_6047
UpperCAmelCase_     							= 25_6145






@require_sentencepiece
@require_tokenizers
class     lowercase__       (	__lowerCamelCase			,				unittest.TestCase				):

   '''simple docstring'''


   a						:							Optional[int]   		=		NllbTokenizer
   a						:							Dict   		=		NllbTokenizerFast
   a						:							List[Any]   		=		True
   a						:							int   		=		True
   a						:							Dict   		=		{}




   def       UpperCamelCase__  (    self							) ->      Tuple:






       """simple docstring"""
       super().setUp()

       # We have a SentencePiece fixture for testing
       UpperCamelCase__							:				Dict          =      NllbTokenizer(__magic_name__,		keep_accents=__magic_name__							)
       tokenizer.save_pretrained(self.tmpdirname							)




   def       UpperCamelCase__  (    self							) ->      Tuple:






       """simple docstring"""
       UpperCamelCase__							:				str          =      NllbTokenizer(__magic_name__,		keep_accents=__magic_name__							)

       UpperCamelCase__							:				int          =      tokenizer.tokenize('''This is a test'''							)
       self.assertListEqual(__magic_name__,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(__magic_name__							),		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]],		)

       UpperCamelCase__							:				List[str]          =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
       self.assertListEqual(
           __magic_name__,		[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ],		)
       UpperCamelCase__							:				List[Any]          =      tokenizer.convert_tokens_to_ids(__magic_name__							)
       self.assertListEqual(
           __magic_name__,		[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ],		)

       UpperCamelCase__							:				str          =      tokenizer.convert_ids_to_tokens(__magic_name__							)
       self.assertListEqual(
           __magic_name__,		[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ],		)




   def       UpperCamelCase__  (    self							) ->      Any:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"							):
               UpperCamelCase__							:				List[Any]          =      self.rust_tokenizer_class.from_pretrained(__magic_name__,		**__magic_name__							)
               UpperCamelCase__							:				Tuple          =      self.tokenizer_class.from_pretrained(__magic_name__,		**__magic_name__							)

               UpperCamelCase__							:				Optional[int]          =      tempfile.mkdtemp()

               UpperCamelCase__							:				int          =      tokenizer_r.save_pretrained(__magic_name__							)
               UpperCamelCase__							:				List[Any]          =      tokenizer_p.save_pretrained(__magic_name__							)

               # Checks it save with the same files + the tokenizer.json file for the fast one
               self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
               UpperCamelCase__							:				List[Any]          =      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
               self.assertSequenceEqual(__magic_name__,		__magic_name__							)

               # Checks everything loads correctly in the same way
               UpperCamelCase__							:				str          =      tokenizer_r.from_pretrained(__magic_name__							)
               UpperCamelCase__							:				Optional[int]          =      tokenizer_p.from_pretrained(__magic_name__							)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                   self.assertTrue(hasattr(__magic_name__,		__magic_name__							)							)

               shutil.rmtree(__magic_name__							)

               # Save tokenizer rust, legacy_format=True
               UpperCamelCase__							:				Optional[Any]          =      tempfile.mkdtemp()

               UpperCamelCase__							:				Tuple          =      tokenizer_r.save_pretrained(__magic_name__,		legacy_format=__magic_name__							)
               UpperCamelCase__							:				List[str]          =      tokenizer_p.save_pretrained(__magic_name__							)

               # Checks it save with the same files
               self.assertSequenceEqual(__magic_name__,		__magic_name__							)

               # Checks everything loads correctly in the same way
               UpperCamelCase__							:				Tuple          =      tokenizer_r.from_pretrained(__magic_name__							)
               UpperCamelCase__							:				List[str]          =      tokenizer_p.from_pretrained(__magic_name__							)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                   self.assertTrue(hasattr(__magic_name__,		__magic_name__							)							)

               shutil.rmtree(__magic_name__							)

               # Save tokenizer rust, legacy_format=False
               UpperCamelCase__							:				int          =      tempfile.mkdtemp()

               UpperCamelCase__							:				Tuple          =      tokenizer_r.save_pretrained(__magic_name__,		legacy_format=__magic_name__							)
               UpperCamelCase__							:				List[str]          =      tokenizer_p.save_pretrained(__magic_name__							)

               # Checks it saved the tokenizer.json file
               self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

               # Checks everything loads correctly in the same way
               UpperCamelCase__							:				Dict          =      tokenizer_r.from_pretrained(__magic_name__							)
               UpperCamelCase__							:				Union[str, Any]          =      tokenizer_p.from_pretrained(__magic_name__							)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                   self.assertTrue(hasattr(__magic_name__,		__magic_name__							)							)

               shutil.rmtree(__magic_name__							)




   @require_torch
   def       UpperCamelCase__  (    self							) ->      int:






       """simple docstring"""
       if not self.test_seqaseq:
           return

       UpperCamelCase__							:				int          =      self.get_tokenizers()
       for tokenizer in tokenizers:
           with self.subTest(f"{tokenizer.__class__.__name__}"							):
               # Longer text that will definitely require truncation.
               UpperCamelCase__							:				str          =      [
                   ''' UN Chief Says There Is No Military Solution in Syria''',
                   ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                   ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                   ''' will only worsen the violence and misery for millions of people.''',
               ]
               UpperCamelCase__							:				List[str]          =      [
                   '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                   '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                   ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                   ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
               ]
               try:
                   UpperCamelCase__							:				Any          =      tokenizer.prepare_seqaseq_batch(
                       src_texts=__magic_name__,		tgt_texts=__magic_name__,		max_length=3,		max_target_length=10,		return_tensors='''pt''',		src_lang='''eng_Latn''',		tgt_lang='''ron_Latn''',		)
               except NotImplementedError:
                   return
               self.assertEqual(batch.input_ids.shape[1],		3							)
               self.assertEqual(batch.labels.shape[1],		10							)
               # max_target_length will default to max_length if not specified
               UpperCamelCase__							:				str          =      tokenizer.prepare_seqaseq_batch(
                   __magic_name__,		tgt_texts=__magic_name__,		max_length=3,		return_tensors='''pt'''							)
               self.assertEqual(batch.input_ids.shape[1],		3							)
               self.assertEqual(batch.labels.shape[1],		3							)

               UpperCamelCase__							:				Dict          =      tokenizer.prepare_seqaseq_batch(
                   src_texts=__magic_name__,		max_length=3,		max_target_length=10,		return_tensors='''pt'''							)
               self.assertEqual(batch_encoder_only.input_ids.shape[1],		3							)
               self.assertEqual(batch_encoder_only.attention_mask.shape[1],		3							)
               self.assertNotIn('''decoder_input_ids''',		__magic_name__							)




   @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''							)
   def       UpperCamelCase__  (    self							) ->      str:






       """simple docstring"""
       pass




   def       UpperCamelCase__  (    self							) ->      Union[str, Any]:






       """simple docstring"""
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"							):
               UpperCamelCase__							:				Dict          =      [AddedToken('''<special>''',		lstrip=__magic_name__							)]

               UpperCamelCase__							:				Optional[Any]          =      self.rust_tokenizer_class.from_pretrained(
                   __magic_name__,		additional_special_tokens=__magic_name__,		**__magic_name__							)
               UpperCamelCase__							:				List[str]          =      tokenizer_r.encode('''Hey this is a <special> token'''							)

               UpperCamelCase__							:				Dict          =      tokenizer_r.encode('''<special>''',		add_special_tokens=__magic_name__							)[0]

               self.assertTrue(special_token_id in r_output							)

               if self.test_slow_tokenizer:
                   UpperCamelCase__							:				Dict          =      self.rust_tokenizer_class.from_pretrained(
                       __magic_name__,		additional_special_tokens=__magic_name__,		**__magic_name__,		)
                   UpperCamelCase__							:				List[str]          =      self.tokenizer_class.from_pretrained(
                       __magic_name__,		additional_special_tokens=__magic_name__,		**__magic_name__							)

                   UpperCamelCase__							:				int          =      tokenizer_p.encode('''Hey this is a <special> token'''							)

                   UpperCamelCase__							:				List[str]          =      tokenizer_cr.encode('''Hey this is a <special> token'''							)

                   self.assertEqual(__magic_name__,		__magic_name__							)
                   self.assertEqual(__magic_name__,		__magic_name__							)
                   self.assertTrue(special_token_id in p_output							)
                   self.assertTrue(special_token_id in cr_output							)






@require_torch
@require_sentencepiece
@require_tokenizers
class     lowercase__       (	unittest.TestCase				):

   '''simple docstring'''


   a						:							Union[str, Any]   		=		"facebook/nllb-200-distilled-600M"
   a						:							str   		=		[
       " UN Chief Says There Is No Military Solution in Syria",
       " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
   ]
   a						:							Optional[int]   		=		[
       "Şeful ONU declară că nu există o soluţie militară în Siria",
       "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
       " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
       " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
   ]
   a						:							Union[str, Any]   		=		[
       25_6047,
       1_6297,
       13_4408,
       8165,
       24_8066,
       1_4734,
       950,
       1135,
       10_5721,
       3573,
       83,
       2_7352,
       108,
       4_9486,
       2,
   ]




   @classmethod
   def       UpperCamelCase__  (    cls							) ->      Optional[int]:






       """simple docstring"""
       UpperCamelCase__							:				NllbTokenizer          =      NllbTokenizer.from_pretrained(
           cls.checkpoint_name,		src_lang='''eng_Latn''',		tgt_lang='''ron_Latn'''							)
       UpperCamelCase__							:				int          =      1
       return cls




   def       UpperCamelCase__  (    self							) ->      Dict:






       """simple docstring"""
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab'''],		256001							)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn'''],		256002							)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn'''],		256057							)




   def       UpperCamelCase__  (    self							) ->      Optional[int]:






       """simple docstring"""
       UpperCamelCase__							:				List[Any]          =      self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
       self.assertListEqual(self.expected_src_tokens,		__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Union[str, Any]:






       """simple docstring"""
       self.assertIn(__magic_name__,		self.tokenizer.all_special_ids							)
       # fmt: off
       UpperCamelCase__							:				Optional[int]          =      [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
       # fmt: on

       UpperCamelCase__							:				List[str]          =      self.tokenizer.decode(__magic_name__,		skip_special_tokens=__magic_name__							)
       UpperCamelCase__							:				str          =      self.tokenizer.decode(generated_ids[1:],		skip_special_tokens=__magic_name__							)
       self.assertEqual(__magic_name__,		__magic_name__							)
       self.assertNotIn(self.tokenizer.eos_token,		__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      ['''this is gunna be a long sentence ''' * 20]
       assert isinstance(src_text[0],		__magic_name__							)
       UpperCamelCase__							:				Any          =      10
       UpperCamelCase__							:				int          =      self.tokenizer(__magic_name__,		max_length=__magic_name__,		truncation=__magic_name__							).input_ids[0]
       self.assertEqual(ids[-1],		2							)
       self.assertEqual(ids[0],		__magic_name__							)
       self.assertEqual(len(__magic_name__							),		__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Optional[Any]:






       """simple docstring"""
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']							),		[256203, 3]							)




   def       UpperCamelCase__  (    self							) ->      Any:






       """simple docstring"""
       UpperCamelCase__							:				int          =      tempfile.mkdtemp()
       UpperCamelCase__							:				Optional[int]          =      self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(__magic_name__							)
       UpperCamelCase__							:				List[Any]          =      NllbTokenizer.from_pretrained(__magic_name__							)
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids,		__magic_name__							)




   @require_torch
   def       UpperCamelCase__  (    self							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				Dict          =      self.tokenizer(
           self.src_text,		text_target=self.tgt_text,		padding=__magic_name__,		truncation=__magic_name__,		max_length=len(self.expected_src_tokens							),		return_tensors='''pt''',		)
       UpperCamelCase__							:				Union[str, Any]          =      shift_tokens_right(
           batch['''labels'''],		self.tokenizer.pad_token_id,		self.tokenizer.lang_code_to_id['''ron_Latn''']							)

       self.assertIsInstance(__magic_name__,		__magic_name__							)

       self.assertEqual((2, 15),		batch.input_ids.shape							)
       self.assertEqual((2, 15),		batch.attention_mask.shape							)
       UpperCamelCase__							:				Union[str, Any]          =      batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens,		__magic_name__							)
       self.assertEqual(__magic_name__,		batch.decoder_input_ids[0, 0]							)  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens,		[EN_CODE]							)
       self.assertEqual(self.tokenizer.suffix_tokens,		[self.tokenizer.eos_token_id]							)




   def       UpperCamelCase__  (    self							) ->      Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:				List[str]          =      self.tokenizer(self.src_text,		padding=__magic_name__,		truncation=__magic_name__,		max_length=3,		return_tensors='''pt'''							)
       UpperCamelCase__							:				Dict          =      self.tokenizer(
           text_target=self.tgt_text,		padding=__magic_name__,		truncation=__magic_name__,		max_length=10,		return_tensors='''pt'''							)
       UpperCamelCase__							:				str          =      targets['''input_ids''']
       UpperCamelCase__							:				List[str]          =      shift_tokens_right(
           __magic_name__,		self.tokenizer.pad_token_id,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang],		)

       self.assertEqual(batch.input_ids.shape[1],		3							)
       self.assertEqual(batch.decoder_input_ids.shape[1],		10							)




   @require_torch
   def       UpperCamelCase__  (    self							) ->      Optional[Any]:






       """simple docstring"""
       UpperCamelCase__							:				str          =      self.tokenizer._build_translation_inputs(
           '''A test''',		return_tensors='''pt''',		src_lang='''eng_Latn''',		tgt_lang='''fra_Latn'''							)

       self.assertEqual(
           nested_simplify(__magic_name__							),		{
               # A, test, EOS, en_XX
               '''input_ids''': [[256047, 70, 7356, 2]],
               '''attention_mask''': [[1, 1, 1, 1]],
               # ar_AR
               '''forced_bos_token_id''': 256057,
           },		)




   @require_torch
   def       UpperCamelCase__  (    self							) ->      Optional[Any]:






       """simple docstring"""
       UpperCamelCase__							:				Optional[int]          =      True
       UpperCamelCase__							:				Dict          =      self.tokenizer(
           '''UN Chief says there is no military solution in Syria''',		src_lang='''eng_Latn''',		tgt_lang='''fra_Latn'''							)
       self.assertEqual(
           inputs.input_ids,		[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]							)

       UpperCamelCase__							:				Any          =      False
       UpperCamelCase__							:				Dict          =      self.tokenizer(
           '''UN Chief says there is no military solution in Syria''',		src_lang='''eng_Latn''',		tgt_lang='''fra_Latn'''							)
       self.assertEqual(
           inputs.input_ids,		[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]							)








from __future__ import annotations




def 		lowerCAmelCase_    (  __UpperCAmelCase:			list[int]		,   __UpperCAmelCase:			int      )      ->	list[int]:
    UpperCamelCase__							:				Optional[int]          =      0
    UpperCamelCase__							:				Tuple          =      len(__UpperCAmelCase      ) - 1

    while i < j:
        if nums[i] + nums[j] == target:
            return [i, j]
        elif nums[i] + nums[j] < target:
            UpperCamelCase__							:				Tuple          =      i + 1
        else:
            UpperCamelCase__							:				str          =      j - 1

    return []


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  print(F'''{two_pointer([2, 7, 11, 15], 9) = }''')

