




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase				:		Any     =   get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCamelCase				:		Dict     =   256_047
lowerCamelCase				:		Tuple     =   256_145







@require_sentencepiece
@require_tokenizers
class   A(						UpperCamelCase							,      unittest.TestCase   ):



 '''simple docstring'''


 UpperCamelCase						   =						NllbTokenizer
 UpperCamelCase						   =						NllbTokenizerFast
 UpperCamelCase						   =						True
 UpperCamelCase						   =						True
 UpperCamelCase						   =						{}

 def 				a__ (				self					:  int  )   ->	str:

       """simple docstring"""






       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCamelCase_		     =    NllbTokenizer(A_						,							keep_accents=A_  )
       tokenizer.save_pretrained(self.tmpdirname  )

 def 				a__ (				self					:  List[str]  )   ->	Dict:

       """simple docstring"""






       lowerCamelCase_		     =    NllbTokenizer(A_						,							keep_accents=A_  )

       lowerCamelCase_		     =    tokenizer.tokenize('This is a test'  )
       self.assertListEqual(A_						,							['▁This', '▁is', '▁a', '▁t', 'est']  )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(A_  )						,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,							)

       lowerCamelCase_		     =    tokenizer.tokenize('I was born in 92000, and this is falsé.'  )
       self.assertListEqual(
           A_						,							[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]						,							)
       lowerCamelCase_		     =    tokenizer.convert_tokens_to_ids(A_  )
       self.assertListEqual(
           A_						,							[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]						,							)

       lowerCamelCase_		     =    tokenizer.convert_ids_to_tokens(A_  )
       self.assertListEqual(
           A_						,							[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '<unk>',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '<unk>',
               '.',
           ]						,							)

 def 				a__ (				self					:  Dict  )   ->	int:

       """simple docstring"""






       lowerCamelCase_		     =    (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
                   lowerCamelCase_		     =    self.rust_tokenizer_class.from_pretrained(A_						,							**A_  )
                   lowerCamelCase_		     =    self.tokenizer_class.from_pretrained(A_						,							**A_  )

                   lowerCamelCase_		     =    tempfile.mkdtemp()

                   lowerCamelCase_		     =    tokenizer_r.save_pretrained(A_  )
                   lowerCamelCase_		     =    tokenizer_p.save_pretrained(A_  )

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files  )  )
                   lowerCamelCase_		     =    tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f  )
                   self.assertSequenceEqual(A_						,							A_  )

                   # Checks everything loads correctly in the same way
                   lowerCamelCase_		     =    tokenizer_r.from_pretrained(A_  )
                   lowerCamelCase_		     =    tokenizer_p.from_pretrained(A_  )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(A_						,							A_  )  )

                   shutil.rmtree(A_  )

                   # Save tokenizer rust, legacy_format=True
                   lowerCamelCase_		     =    tempfile.mkdtemp()

                   lowerCamelCase_		     =    tokenizer_r.save_pretrained(A_						,							legacy_format=A_  )
                   lowerCamelCase_		     =    tokenizer_p.save_pretrained(A_  )

                   # Checks it save with the same files
                   self.assertSequenceEqual(A_						,							A_  )

                   # Checks everything loads correctly in the same way
                   lowerCamelCase_		     =    tokenizer_r.from_pretrained(A_  )
                   lowerCamelCase_		     =    tokenizer_p.from_pretrained(A_  )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(A_						,							A_  )  )

                   shutil.rmtree(A_  )

                   # Save tokenizer rust, legacy_format=False
                   lowerCamelCase_		     =    tempfile.mkdtemp()

                   lowerCamelCase_		     =    tokenizer_r.save_pretrained(A_						,							legacy_format=A_  )
                   lowerCamelCase_		     =    tokenizer_p.save_pretrained(A_  )

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files  )  )

                   # Checks everything loads correctly in the same way
                   lowerCamelCase_		     =    tokenizer_r.from_pretrained(A_  )
                   lowerCamelCase_		     =    tokenizer_p.from_pretrained(A_  )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(A_						,							A_  )  )

                   shutil.rmtree(A_  )

 @require_torch
 def 				a__ (				self					:  List[Any]  )   ->	Union[str, Any]:

       """simple docstring"""






       if not self.test_seqaseq:
             return

       lowerCamelCase_		     =    self.get_tokenizers()
       for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""  ):
                   # Longer text that will definitely require truncation.
                   lowerCamelCase_		     =    [
                       ' UN Chief Says There Is No Military Solution in Syria',
                       ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                       ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                       ' will only worsen the violence and misery for millions of people.',
                   ]
                   lowerCamelCase_		     =    [
                       'Şeful ONU declară că nu există o soluţie militară în Siria',
                       'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                       ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                       ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                   ]
                   try:
                         lowerCamelCase_		     =    tokenizer.prepare_seqaseq_batch(
                             src_texts=A_						,							tgt_texts=A_						,							max_length=3						,							max_target_length=10						,							return_tensors='pt'						,							src_lang='eng_Latn'						,							tgt_lang='ron_Latn'						,							)
                   except NotImplementedError:
                         return
                   self.assertEqual(batch.input_ids.shape[1]						,							3  )
                   self.assertEqual(batch.labels.shape[1]						,							10  )
                   # max_target_length will default to max_length if not specified
                   lowerCamelCase_		     =    tokenizer.prepare_seqaseq_batch(
                       A_						,							tgt_texts=A_						,							max_length=3						,							return_tensors='pt'  )
                   self.assertEqual(batch.input_ids.shape[1]						,							3  )
                   self.assertEqual(batch.labels.shape[1]						,							3  )

                   lowerCamelCase_		     =    tokenizer.prepare_seqaseq_batch(
                       src_texts=A_						,							max_length=3						,							max_target_length=10						,							return_tensors='pt'  )
                   self.assertEqual(batch_encoder_only.input_ids.shape[1]						,							3  )
                   self.assertEqual(batch_encoder_only.attention_mask.shape[1]						,							3  )
                   self.assertNotIn('decoder_input_ids'						,							A_  )

 @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'  )
 def 				a__ (				self					:  List[Any]  )   ->	List[str]:

       """simple docstring"""






       pass





 def 				a__ (				self					:  List[str]  )   ->	Any:

       """simple docstring"""






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
                   lowerCamelCase_		     =    [AddedToken('<special>'						,							lstrip=A_  )]

                   lowerCamelCase_		     =    self.rust_tokenizer_class.from_pretrained(
                       A_						,							additional_special_tokens=A_						,							**A_  )
                   lowerCamelCase_		     =    tokenizer_r.encode('Hey this is a <special> token'  )

                   lowerCamelCase_		     =    tokenizer_r.encode('<special>'						,							add_special_tokens=A_  )[0]

                   self.assertTrue(special_token_id in r_output  )

                   if self.test_slow_tokenizer:
                         lowerCamelCase_		     =    self.rust_tokenizer_class.from_pretrained(
                             A_						,							additional_special_tokens=A_						,							**A_						,							)
                         lowerCamelCase_		     =    self.tokenizer_class.from_pretrained(
                             A_						,							additional_special_tokens=A_						,							**A_  )

                         lowerCamelCase_		     =    tokenizer_p.encode('Hey this is a <special> token'  )

                         lowerCamelCase_		     =    tokenizer_cr.encode('Hey this is a <special> token'  )

                         self.assertEqual(A_						,							A_  )
                         self.assertEqual(A_						,							A_  )
                         self.assertTrue(special_token_id in p_output  )
                         self.assertTrue(special_token_id in cr_output  )







@require_torch
@require_sentencepiece
@require_tokenizers
class   A(						unittest.TestCase   ):



 '''simple docstring'''


 UpperCamelCase						   =						'''facebook/nllb-200-distilled-600M'''
 UpperCamelCase						   =						[
     ''' UN Chief Says There Is No Military Solution in Syria''',
     ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
 ]
 UpperCamelCase						   =						[
     '''Şeful ONU declară că nu există o soluţie militară în Siria''',
     '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
     ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
     ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
 ]
 UpperCamelCase						   =						[
     25_6047,
     1_6297,
     13_4408,
     8165,
     24_8066,
     1_4734,
     950,
     1135,
     10_5721,
     3573,
     83,
     2_7352,
     108,
     4_9486,
     2,
 ]

 @classmethod
 def 				a__ (				cls					:  Any  )   ->	Dict:

       """simple docstring"""






       lowerCamelCase_		     =    NllbTokenizer.from_pretrained(
           cls.checkpoint_name						,							src_lang='eng_Latn'						,							tgt_lang='ron_Latn'  )
       lowerCamelCase_		     =    1
       return cls

 def 				a__ (				self					:  Optional[int]  )   ->	Optional[Any]:

       """simple docstring"""






       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']						,							256001  )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']						,							256002  )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']						,							256057  )

 def 				a__ (				self					:  Tuple  )   ->	str:

       """simple docstring"""






       lowerCamelCase_		     =    self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
       self.assertListEqual(self.expected_src_tokens						,							A_  )

 def 				a__ (				self					:  Union[str, Any]  )   ->	List[Any]:

       """simple docstring"""






       self.assertIn(A_						,							self.tokenizer.all_special_ids  )
       # fmt: off
       lowerCamelCase_		     =    [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
       # fmt: on

       lowerCamelCase_		     =    self.tokenizer.decode(A_						,							skip_special_tokens=A_  )
       lowerCamelCase_		     =    self.tokenizer.decode(generated_ids[1:]						,							skip_special_tokens=A_  )
       self.assertEqual(A_						,							A_  )
       self.assertNotIn(self.tokenizer.eos_token						,							A_  )

 def 				a__ (				self					:  int  )   ->	List[str]:

       """simple docstring"""






       lowerCamelCase_		     =    ['this is gunna be a long sentence ' * 20]
       assert isinstance(src_text[0]						,							A_  )
       lowerCamelCase_		     =    10
       lowerCamelCase_		     =    self.tokenizer(A_						,							max_length=A_						,							truncation=A_  ).input_ids[0]
       self.assertEqual(ids[-1]						,							2  )
       self.assertEqual(ids[0]						,							A_  )
       self.assertEqual(len(A_  )						,							A_  )

 def 				a__ (				self					:  int  )   ->	str:

       """simple docstring"""






       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']  )						,							[256203, 3]  )

 def 				a__ (				self					:  Union[str, Any]  )   ->	Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_		     =    tempfile.mkdtemp()
       lowerCamelCase_		     =    self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(A_  )
       lowerCamelCase_		     =    NllbTokenizer.from_pretrained(A_  )
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,							A_  )

 @require_torch
 def 				a__ (				self					:  List[str]  )   ->	int:

       """simple docstring"""






       lowerCamelCase_		     =    self.tokenizer(
           self.src_text						,							text_target=self.tgt_text						,							padding=A_						,							truncation=A_						,							max_length=len(self.expected_src_tokens  )						,							return_tensors='pt'						,							)
       lowerCamelCase_		     =    shift_tokens_right(
           batch['labels']						,							self.tokenizer.pad_token_id						,							self.tokenizer.lang_code_to_id['ron_Latn']  )

       self.assertIsInstance(A_						,							A_  )

       self.assertEqual((2, 15)						,							batch.input_ids.shape  )
       self.assertEqual((2, 15)						,							batch.attention_mask.shape  )
       lowerCamelCase_		     =    batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens						,							A_  )
       self.assertEqual(A_						,							batch.decoder_input_ids[0, 0]  )  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens						,							[EN_CODE]  )
       self.assertEqual(self.tokenizer.suffix_tokens						,							[self.tokenizer.eos_token_id]  )

 def 				a__ (				self					:  List[str]  )   ->	Optional[int]:

       """simple docstring"""






       lowerCamelCase_		     =    self.tokenizer(self.src_text						,							padding=A_						,							truncation=A_						,							max_length=3						,							return_tensors='pt'  )
       lowerCamelCase_		     =    self.tokenizer(
           text_target=self.tgt_text						,							padding=A_						,							truncation=A_						,							max_length=10						,							return_tensors='pt'  )
       lowerCamelCase_		     =    targets['input_ids']
       lowerCamelCase_		     =    shift_tokens_right(
           A_						,							self.tokenizer.pad_token_id						,							decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]						,							)

       self.assertEqual(batch.input_ids.shape[1]						,							3  )
       self.assertEqual(batch.decoder_input_ids.shape[1]						,							10  )

 @require_torch
 def 				a__ (				self					:  Tuple  )   ->	Optional[Any]:

       """simple docstring"""






       lowerCamelCase_		     =    self.tokenizer._build_translation_inputs(
           'A test'						,							return_tensors='pt'						,							src_lang='eng_Latn'						,							tgt_lang='fra_Latn'  )

       self.assertEqual(
           nested_simplify(A_  )						,							{
               # A, test, EOS, en_XX
               'input_ids': [[256047, 70, 7356, 2]],
               'attention_mask': [[1, 1, 1, 1]],
               # ar_AR
               'forced_bos_token_id': 256057,
           }						,							)





 @require_torch
 def 				a__ (				self					:  Any  )   ->	Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    self.tokenizer(
           'UN Chief says there is no military solution in Syria'						,							src_lang='eng_Latn'						,							tgt_lang='fra_Latn'  )
       self.assertEqual(
           inputs.input_ids						,							[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]  )

       lowerCamelCase_		     =    False
       lowerCamelCase_		     =    self.tokenizer(
           'UN Chief says there is no military solution in Syria'						,							src_lang='eng_Latn'						,							tgt_lang='fra_Latn'  )
       self.assertEqual(
           inputs.input_ids						,							[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]  )









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase				:		Union[str, Any]     =   {
    "configuration_convbert": ["CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConvBertConfig", "ConvBertOnnxConfig"],
    "tokenization_convbert": ["ConvBertTokenizer"],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Optional[Any]     =   ["ConvBertTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Dict     =   [
     "CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "ConvBertForMaskedLM",
     "ConvBertForMultipleChoice",
     "ConvBertForQuestionAnswering",
     "ConvBertForSequenceClassification",
     "ConvBertForTokenClassification",
     "ConvBertLayer",
     "ConvBertModel",
     "ConvBertPreTrainedModel",
     "load_tf_weights_in_convbert",
 ]


try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Tuple     =   [
     "TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFConvBertForMaskedLM",
     "TFConvBertForMultipleChoice",
     "TFConvBertForQuestionAnswering",
     "TFConvBertForSequenceClassification",
     "TFConvBertForTokenClassification",
     "TFConvBertLayer",
     "TFConvBertModel",
     "TFConvBertPreTrainedModel",
 ]


if TYPE_CHECKING:
 from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
 from .tokenization_convbert import ConvBertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_convbert_fast import ConvBertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_convbert import (
      CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      ConvBertForMaskedLM,
      ConvBertForMultipleChoice,
      ConvBertForQuestionAnswering,
      ConvBertForSequenceClassification,
      ConvBertForTokenClassification,
      ConvBertLayer,
      ConvBertModel,
      ConvBertPreTrainedModel,
      load_tf_weights_in_convbert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_convbert import (
      TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertLayer,
      TFConvBertModel,
      TFConvBertPreTrainedModel,
  )


else:
 import sys

 lowerCamelCase				:		Optional[int]     =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



