




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
					from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCamelCase__    			=			256047
UpperCamelCase__    			=			256145
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    Union[str, Any]					=    NllbTokenizer
						_a							:    Optional[int]					=    NllbTokenizerFast
						_a							:    Optional[int]					=    True
						_a							:    Dict					=    True
						_a							:    List[Any]					=    {}






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       NllbTokenizer(A_		,			keep_accents=A_			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       NllbTokenizer(A_		,			keep_accents=A_			)

											__lowerCAmelCase				   =       tokenizer.tokenize("This is a test"			)
											self.assertListEqual(A_		,			["▁This", "▁is", "▁a", "▁t", "est"]			)

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(A_			)		,			[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]		,			)

											__lowerCAmelCase				   =       tokenizer.tokenize("I was born in 92000, and this is falsé."			)
											self.assertListEqual(
											    A_		,			[
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "9",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "é",
											        ".",
											    ]		,			)
											__lowerCAmelCase				   =       tokenizer.convert_tokens_to_ids(A_			)
											self.assertListEqual(
											    A_		,			[
											        value + tokenizer.fairseq_offset
											        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
											    ]		,			)

											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(A_			)
											self.assertListEqual(
											    A_		,			[
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "<unk>",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "<unk>",
											        ".",
											    ]		,			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																					__lowerCAmelCase				   =       self.rust_tokenizer_class.from_pretrained(A_		,			**A_			)
																					__lowerCAmelCase				   =       self.tokenizer_class.from_pretrained(A_		,			**A_			)

																					__lowerCAmelCase				   =       tempfile.mkdtemp()

																					__lowerCAmelCase				   =       tokenizer_r.save_pretrained(A_			)
																					__lowerCAmelCase				   =       tokenizer_p.save_pretrained(A_			)

																					# Checks it save with the same files + the tokenizer.json file for the fast one
																					self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files			)			)
																					__lowerCAmelCase				   =       tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f			)
																					self.assertSequenceEqual(A_		,			A_			)

																					# Checks everything loads correctly in the same way
																					__lowerCAmelCase				   =       tokenizer_r.from_pretrained(A_			)
																					__lowerCAmelCase				   =       tokenizer_p.from_pretrained(A_			)

																					# Check special tokens are set accordingly on Rust and Python
																					for key in tokenizer_pp.special_tokens_map:
																										self.assertTrue(hasattr(A_		,			A_			)			)

																					shutil.rmtree(A_			)

																					# Save tokenizer rust, legacy_format=True
																					__lowerCAmelCase				   =       tempfile.mkdtemp()

																					__lowerCAmelCase				   =       tokenizer_r.save_pretrained(A_		,			legacy_format=A_			)
																					__lowerCAmelCase				   =       tokenizer_p.save_pretrained(A_			)

																					# Checks it save with the same files
																					self.assertSequenceEqual(A_		,			A_			)

																					# Checks everything loads correctly in the same way
																					__lowerCAmelCase				   =       tokenizer_r.from_pretrained(A_			)
																					__lowerCAmelCase				   =       tokenizer_p.from_pretrained(A_			)

																					# Check special tokens are set accordingly on Rust and Python
																					for key in tokenizer_pp.special_tokens_map:
																										self.assertTrue(hasattr(A_		,			A_			)			)

																					shutil.rmtree(A_			)

																					# Save tokenizer rust, legacy_format=False
																					__lowerCAmelCase				   =       tempfile.mkdtemp()

																					__lowerCAmelCase				   =       tokenizer_r.save_pretrained(A_		,			legacy_format=A_			)
																					__lowerCAmelCase				   =       tokenizer_p.save_pretrained(A_			)

																					# Checks it saved the tokenizer.json file
																					self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files			)			)

																					# Checks everything loads correctly in the same way
																					__lowerCAmelCase				   =       tokenizer_r.from_pretrained(A_			)
																					__lowerCAmelCase				   =       tokenizer_p.from_pretrained(A_			)

																					# Check special tokens are set accordingly on Rust and Python
																					for key in tokenizer_pp.special_tokens_map:
																										self.assertTrue(hasattr(A_		,			A_			)			)

																					shutil.rmtree(A_			)






						@require_torch
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											if not self.test_seqaseq:
																return

											__lowerCAmelCase				   =       self.get_tokenizers()
											for tokenizer in tokenizers:
																with self.subTest(f"""{tokenizer.__class__.__name__}"""			):
																					# Longer text that will definitely require truncation.
																					__lowerCAmelCase				   =       [
																					    " UN Chief Says There Is No Military Solution in Syria",
																					    " Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for"
																					    " Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons"
																					    " will only worsen the violence and misery for millions of people.",
																					]
																					__lowerCAmelCase				   =       [
																					    "Şeful ONU declară că nu există o soluţie militară în Siria",
																					    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
																					    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
																					    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
																					]
																					try:
																										__lowerCAmelCase				   =       tokenizer.prepare_seqaseq_batch(
																										    src_texts=A_		,			tgt_texts=A_		,			max_length=3		,			max_target_length=1_0		,			return_tensors="pt"		,			src_lang="eng_Latn"		,			tgt_lang="ron_Latn"		,			)
																					except NotImplementedError:
																										return
																					self.assertEqual(batch.input_ids.shape[1]		,			3			)
																					self.assertEqual(batch.labels.shape[1]		,			1_0			)
																					# max_target_length will default to max_length if not specified
																					__lowerCAmelCase				   =       tokenizer.prepare_seqaseq_batch(
																					    A_		,			tgt_texts=A_		,			max_length=3		,			return_tensors="pt"			)
																					self.assertEqual(batch.input_ids.shape[1]		,			3			)
																					self.assertEqual(batch.labels.shape[1]		,			3			)

																					__lowerCAmelCase				   =       tokenizer.prepare_seqaseq_batch(
																					    src_texts=A_		,			max_length=3		,			max_target_length=1_0		,			return_tensors="pt"			)
																					self.assertEqual(batch_encoder_only.input_ids.shape[1]		,			3			)
																					self.assertEqual(batch_encoder_only.attention_mask.shape[1]		,			3			)
																					self.assertNotIn("decoder_input_ids"		,			A_			)






						@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																					__lowerCAmelCase				   =       [AddedToken("<special>"		,			lstrip=A_			)]

																					__lowerCAmelCase				   =       self.rust_tokenizer_class.from_pretrained(
																					    A_		,			additional_special_tokens=A_		,			**A_			)
																					__lowerCAmelCase				   =       tokenizer_r.encode("Hey this is a <special> token"			)

																					__lowerCAmelCase				   =       tokenizer_r.encode("<special>"		,			add_special_tokens=A_			)[0]

																					self.assertTrue(special_token_id in r_output			)

																					if self.test_slow_tokenizer:
																										__lowerCAmelCase				   =       self.rust_tokenizer_class.from_pretrained(
																										    A_		,			additional_special_tokens=A_		,			**A_		,			)
																										__lowerCAmelCase				   =       self.tokenizer_class.from_pretrained(
																										    A_		,			additional_special_tokens=A_		,			**A_			)

																										__lowerCAmelCase				   =       tokenizer_p.encode("Hey this is a <special> token"			)

																										__lowerCAmelCase				   =       tokenizer_cr.encode("Hey this is a <special> token"			)

																										self.assertEqual(A_		,			A_			)
																										self.assertEqual(A_		,			A_			)
																										self.assertTrue(special_token_id in p_output			)
																										self.assertTrue(special_token_id in cr_output			)






@require_torch
@require_sentencepiece
@require_tokenizers
class 			a__    (					unittest.TestCase      ):
						_a							:    Any					=    """facebook/nllb-200-distilled-600M"""
						_a							:    List[Any]					=    [
						    """ UN Chief Says There Is No Military Solution in Syria""",
						    """ Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
						]
						_a							:    Any					=    [
						    """Şeful ONU declară că nu există o soluţie militară în Siria""",
						    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
						    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
						    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
						]
						_a							:    Tuple					=    [
						    2_5_6_0_4_7,
						    1_6_2_9_7,
						    1_3_4_4_0_8,
						    8_1_6_5,
						    2_4_8_0_6_6,
						    1_4_7_3_4,
						    9_5_0,
						    1_1_3_5,
						    1_0_5_7_2_1,
						    3_5_7_3,
						    8_3,
						    2_7_3_5_2,
						    1_0_8,
						    4_9_4_8_6,
						    2,
						]






						@classmethod
						def 				__SCREAMING_SNAKE_CASE(   cls			):




											"""simple docstring"""



											__lowerCAmelCase				   =       NllbTokenizer.from_pretrained(
											    cls.checkpoint_name		,			src_lang="eng_Latn"		,			tgt_lang="ron_Latn"			)
											__lowerCAmelCase				   =       1
											return cls






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]		,			2_5_6_0_0_1			)
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]		,			2_5_6_0_0_2			)
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]		,			2_5_6_0_5_7			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
											self.assertListEqual(self.expected_src_tokens		,			A_			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertIn(A_		,			self.tokenizer.all_special_ids			)
											# fmt: off
											__lowerCAmelCase				   =       [RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
											# fmt: on

											__lowerCAmelCase				   =       self.tokenizer.decode(A_		,			skip_special_tokens=A_			)
											__lowerCAmelCase				   =       self.tokenizer.decode(generated_ids[1:]		,			skip_special_tokens=A_			)
											self.assertEqual(A_		,			A_			)
											self.assertNotIn(self.tokenizer.eos_token		,			A_			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       ["this is gunna be a long sentence " * 2_0]
											assert isinstance(src_text[0]		,			A_			)
											__lowerCAmelCase				   =       1_0
											__lowerCAmelCase				   =       self.tokenizer(A_		,			max_length=A_		,			truncation=A_			).input_ids[0]
											self.assertEqual(ids[-1]		,			2			)
											self.assertEqual(ids[0]		,			A_			)
											self.assertEqual(len(A_			)		,			A_			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]			)		,			[2_5_6_2_0_3, 3]			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       tempfile.mkdtemp()
											__lowerCAmelCase				   =       self.tokenizer.fairseq_tokens_to_ids
											self.tokenizer.save_pretrained(A_			)
											__lowerCAmelCase				   =       NllbTokenizer.from_pretrained(A_			)
											self.assertDictEqual(new_tok.fairseq_tokens_to_ids		,			A_			)






						@require_torch
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.tokenizer(
											    self.src_text		,			text_target=self.tgt_text		,			padding=A_		,			truncation=A_		,			max_length=len(self.expected_src_tokens			)		,			return_tensors="pt"		,			)
											__lowerCAmelCase				   =       shift_tokens_right(
											    batch["labels"]		,			self.tokenizer.pad_token_id		,			self.tokenizer.lang_code_to_id["ron_Latn"]			)

											self.assertIsInstance(A_		,			A_			)

											self.assertEqual((2, 1_5)		,			batch.input_ids.shape			)
											self.assertEqual((2, 1_5)		,			batch.attention_mask.shape			)
											__lowerCAmelCase				   =       batch.input_ids.tolist()[0]
											self.assertListEqual(self.expected_src_tokens		,			A_			)
											self.assertEqual(A_		,			batch.decoder_input_ids[0, 0]			)  # EOS
											# Test that special tokens are reset
											self.assertEqual(self.tokenizer.prefix_tokens		,			[EN_CODE]			)
											self.assertEqual(self.tokenizer.suffix_tokens		,			[self.tokenizer.eos_token_id]			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.tokenizer(self.src_text		,			padding=A_		,			truncation=A_		,			max_length=3		,			return_tensors="pt"			)
											__lowerCAmelCase				   =       self.tokenizer(
											    text_target=self.tgt_text		,			padding=A_		,			truncation=A_		,			max_length=1_0		,			return_tensors="pt"			)
											__lowerCAmelCase				   =       targets["input_ids"]
											__lowerCAmelCase				   =       shift_tokens_right(
											    A_		,			self.tokenizer.pad_token_id		,			decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]		,			)

											self.assertEqual(batch.input_ids.shape[1]		,			3			)
											self.assertEqual(batch.decoder_input_ids.shape[1]		,			1_0			)






						@require_torch
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.tokenizer._build_translation_inputs(
											    "A test"		,			return_tensors="pt"		,			src_lang="eng_Latn"		,			tgt_lang="fra_Latn"			)

											self.assertEqual(
											    nested_simplify(A_			)		,			{
											        # A, test, EOS, en_XX
											        "input_ids": [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
											        "attention_mask": [[1, 1, 1, 1]],
											        # ar_AR
											        "forced_bos_token_id": 2_5_6_0_5_7,
											    }		,			)






						@require_torch
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       True
											__lowerCAmelCase				   =       self.tokenizer(
											    "UN Chief says there is no military solution in Syria"		,			src_lang="eng_Latn"		,			tgt_lang="fra_Latn"			)
											self.assertEqual(
											    inputs.input_ids		,			[1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]			)

											__lowerCAmelCase				   =       False
											__lowerCAmelCase				   =       self.tokenizer(
											    "UN Chief says there is no military solution in Syria"		,			src_lang="eng_Latn"		,			tgt_lang="fra_Latn"			)
											self.assertEqual(
											    inputs.input_ids		,			[2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]			)










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase				:		Union[str, Any]     =   {
    "configuration_convbert": ["CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConvBertConfig", "ConvBertOnnxConfig"],
    "tokenization_convbert": ["ConvBertTokenizer"],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Optional[Any]     =   ["ConvBertTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Dict     =   [
     "CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "ConvBertForMaskedLM",
     "ConvBertForMultipleChoice",
     "ConvBertForQuestionAnswering",
     "ConvBertForSequenceClassification",
     "ConvBertForTokenClassification",
     "ConvBertLayer",
     "ConvBertModel",
     "ConvBertPreTrainedModel",
     "load_tf_weights_in_convbert",
 ]


try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Tuple     =   [
     "TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFConvBertForMaskedLM",
     "TFConvBertForMultipleChoice",
     "TFConvBertForQuestionAnswering",
     "TFConvBertForSequenceClassification",
     "TFConvBertForTokenClassification",
     "TFConvBertLayer",
     "TFConvBertModel",
     "TFConvBertPreTrainedModel",
 ]


if TYPE_CHECKING:
 from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
 from .tokenization_convbert import ConvBertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_convbert_fast import ConvBertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_convbert import (
      CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      ConvBertForMaskedLM,
      ConvBertForMultipleChoice,
      ConvBertForQuestionAnswering,
      ConvBertForSequenceClassification,
      ConvBertForTokenClassification,
      ConvBertLayer,
      ConvBertModel,
      ConvBertPreTrainedModel,
      load_tf_weights_in_convbert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_convbert import (
      TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertLayer,
      TFConvBertModel,
      TFConvBertPreTrainedModel,
  )


else:
 import sys

 lowerCamelCase				:		Optional[int]     =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



