


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case				:    Optional[int]       				=     get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
     from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

snake_case				:    str       				=     25_60_47
snake_case				:    List[Any]       				=     25_61_45







@require_sentencepiece
@require_tokenizers
class 		_snake_case  (      _snake_case						,    unittest.TestCase							):
 SCREAMING_SNAKE_CASE__      =       NllbTokenizer
 SCREAMING_SNAKE_CASE__      =       NllbTokenizerFast
 SCREAMING_SNAKE_CASE__      =       True
 SCREAMING_SNAKE_CASE__      =       True
 SCREAMING_SNAKE_CASE__      =       {}




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       a						:List[Any] 							=       NllbTokenizer(_lowerCamelCase					,  keep_accents=_lowerCamelCase    )
       tokenizer.save_pretrained(self.tmpdirname    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       NllbTokenizer(_lowerCamelCase					,  keep_accents=_lowerCamelCase    )

       a						:List[Any] 							=       tokenizer.tokenize('''This is a test'''    )
       self.assertListEqual(_lowerCamelCase					,  ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']    )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_lowerCamelCase    )					,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,  )

       a						:Any 							=       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''    )
       self.assertListEqual(
           _lowerCamelCase					,  [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]					,  )
       a						:Any 							=       tokenizer.convert_tokens_to_ids(_lowerCamelCase    )
       self.assertListEqual(
           _lowerCamelCase					,  [
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]					,  )

       a						:Any 							=       tokenizer.convert_ids_to_tokens(_lowerCamelCase    )
       self.assertListEqual(
           _lowerCamelCase					,  [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ]					,  )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
                   a						:Any 							=       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase					,  **_lowerCamelCase    )
                   a						:Any 							=       self.tokenizer_class.from_pretrained(_lowerCamelCase					,  **_lowerCamelCase    )

                   a						:List[Any] 							=       tempfile.mkdtemp()

                   a						:List[Any] 							=       tokenizer_r.save_pretrained(_lowerCamelCase    )
                   a						:Optional[int] 							=       tokenizer_p.save_pretrained(_lowerCamelCase    )

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files    )    )
                   a						:List[str] 							=       tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f    )
                   self.assertSequenceEqual(_lowerCamelCase					,  _lowerCamelCase    )

                   # Checks everything loads correctly in the same way
                   a						:str 							=       tokenizer_r.from_pretrained(_lowerCamelCase    )
                   a						:str 							=       tokenizer_p.from_pretrained(_lowerCamelCase    )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(_lowerCamelCase					,  _lowerCamelCase    )    )

                   shutil.rmtree(_lowerCamelCase    )

                   # Save tokenizer rust, legacy_format=True
                   a						:int 							=       tempfile.mkdtemp()

                   a						:str 							=       tokenizer_r.save_pretrained(_lowerCamelCase					,  legacy_format=_lowerCamelCase    )
                   a						:List[Any] 							=       tokenizer_p.save_pretrained(_lowerCamelCase    )

                   # Checks it save with the same files
                   self.assertSequenceEqual(_lowerCamelCase					,  _lowerCamelCase    )

                   # Checks everything loads correctly in the same way
                   a						:List[Any] 							=       tokenizer_r.from_pretrained(_lowerCamelCase    )
                   a						:Optional[int] 							=       tokenizer_p.from_pretrained(_lowerCamelCase    )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(_lowerCamelCase					,  _lowerCamelCase    )    )

                   shutil.rmtree(_lowerCamelCase    )

                   # Save tokenizer rust, legacy_format=False
                   a						:Optional[int] 							=       tempfile.mkdtemp()

                   a						:str 							=       tokenizer_r.save_pretrained(_lowerCamelCase					,  legacy_format=_lowerCamelCase    )
                   a						:List[str] 							=       tokenizer_p.save_pretrained(_lowerCamelCase    )

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files    )    )

                   # Checks everything loads correctly in the same way
                   a						:Tuple 							=       tokenizer_r.from_pretrained(_lowerCamelCase    )
                   a						:Optional[Any] 							=       tokenizer_p.from_pretrained(_lowerCamelCase    )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                         self.assertTrue(hasattr(_lowerCamelCase					,  _lowerCamelCase    )    )

                   shutil.rmtree(_lowerCamelCase    )




 @require_torch
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       if not self.test_seqaseq:
             return

       a						:Union[str, Any] 							=       self.get_tokenizers()
       for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
                   # Longer text that will definitely require truncation.
                   a						:int 							=       [
                       ''' UN Chief Says There Is No Military Solution in Syria''',
                       ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                       ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                       ''' will only worsen the violence and misery for millions of people.''',
                   ]
                   a						:Tuple 							=       [
                       '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                       '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                       ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                       ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                   ]
                   try:
                         a						:Optional[Any] 							=       tokenizer.prepare_seqaseq_batch(
                             src_texts=_lowerCamelCase					,  tgt_texts=_lowerCamelCase					,  max_length=3					,  max_target_length=10					,  return_tensors='''pt'''					,  src_lang='''eng_Latn'''					,  tgt_lang='''ron_Latn'''					,  )
                   except NotImplementedError:
                         return
                   self.assertEqual(batch.input_ids.shape[1]					,  3    )
                   self.assertEqual(batch.labels.shape[1]					,  10    )
                   # max_target_length will default to max_length if not specified
                   a						:List[Any] 							=       tokenizer.prepare_seqaseq_batch(
                       _lowerCamelCase					,  tgt_texts=_lowerCamelCase					,  max_length=3					,  return_tensors='''pt'''    )
                   self.assertEqual(batch.input_ids.shape[1]					,  3    )
                   self.assertEqual(batch.labels.shape[1]					,  3    )

                   a						:int 							=       tokenizer.prepare_seqaseq_batch(
                       src_texts=_lowerCamelCase					,  max_length=3					,  max_target_length=10					,  return_tensors='''pt'''    )
                   self.assertEqual(batch_encoder_only.input_ids.shape[1]					,  3    )
                   self.assertEqual(batch_encoder_only.attention_mask.shape[1]					,  3    )
                   self.assertNotIn('''decoder_input_ids'''					,  _lowerCamelCase    )




 @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''    )
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       pass




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
                   a						:Optional[Any] 							=       [AddedToken('''<special>'''					,  lstrip=_lowerCamelCase    )]

                   a						:List[Any] 							=       self.rust_tokenizer_class.from_pretrained(
                       _lowerCamelCase					,  additional_special_tokens=_lowerCamelCase					,  **_lowerCamelCase    )
                   a						:List[str] 							=       tokenizer_r.encode('''Hey this is a <special> token'''    )

                   a						:Optional[Any] 							=       tokenizer_r.encode('''<special>'''					,  add_special_tokens=_lowerCamelCase    )[0]

                   self.assertTrue(special_token_id in r_output    )

                   if self.test_slow_tokenizer:
                         a						:int 							=       self.rust_tokenizer_class.from_pretrained(
                             _lowerCamelCase					,  additional_special_tokens=_lowerCamelCase					,  **_lowerCamelCase					,  )
                         a						:str 							=       self.tokenizer_class.from_pretrained(
                             _lowerCamelCase					,  additional_special_tokens=_lowerCamelCase					,  **_lowerCamelCase    )

                         a						:Any 							=       tokenizer_p.encode('''Hey this is a <special> token'''    )

                         a						:Tuple 							=       tokenizer_cr.encode('''Hey this is a <special> token'''    )

                         self.assertEqual(_lowerCamelCase					,  _lowerCamelCase    )
                         self.assertEqual(_lowerCamelCase					,  _lowerCamelCase    )
                         self.assertTrue(special_token_id in p_output    )
                         self.assertTrue(special_token_id in cr_output    )







@require_torch
@require_sentencepiece
@require_tokenizers
class 		_snake_case  (      unittest.TestCase							):
 SCREAMING_SNAKE_CASE__      =       'facebook/nllb-200-distilled-600M'
 SCREAMING_SNAKE_CASE__      =       [
     ' UN Chief Says There Is No Military Solution in Syria',
     ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
 ]
 SCREAMING_SNAKE_CASE__      =       [
     'Şeful ONU declară că nu există o soluţie militară în Siria',
     'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
     ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
     ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
 ]
 SCREAMING_SNAKE_CASE__      =       [
     25_6047,
     1_6297,
     13_4408,
     8165,
     24_8066,
     1_4734,
     950,
     1135,
     10_5721,
     3573,
     83,
     2_7352,
     108,
     4_9486,
     2,
 ]




 @classmethod
 def        SCREAMING_SNAKE_CASE__				(		cls    ):
       a						:NllbTokenizer 							=       NllbTokenizer.from_pretrained(
           cls.checkpoint_name					,  src_lang='''eng_Latn'''					,  tgt_lang='''ron_Latn'''    )
       a						:Union[str, Any] 							=       1
       return cls




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']					,  25_6001    )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']					,  25_6002    )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']					,  25_6057    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
       self.assertListEqual(self.expected_src_tokens					,  _lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       self.assertIn(_lowerCamelCase					,  self.tokenizer.all_special_ids    )
       # fmt: off
       a						:Union[str, Any] 							=       [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
       # fmt: on

       a						:List[Any] 							=       self.tokenizer.decode(_lowerCamelCase					,  skip_special_tokens=_lowerCamelCase    )
       a						:Any 							=       self.tokenizer.decode(generated_ids[1:]					,  skip_special_tokens=_lowerCamelCase    )
       self.assertEqual(_lowerCamelCase					,  _lowerCamelCase    )
       self.assertNotIn(self.tokenizer.eos_token					,  _lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Optional[int] 							=       ['''this is gunna be a long sentence ''' * 20]
       assert isinstance(src_text[0]					,  _lowerCamelCase    )
       a						:Dict 							=       10
       a						:int 							=       self.tokenizer(_lowerCamelCase					,  max_length=_lowerCamelCase					,  truncation=_lowerCamelCase    ).input_ids[0]
       self.assertEqual(ids[-1]					,  2    )
       self.assertEqual(ids[0]					,  _lowerCamelCase    )
       self.assertEqual(len(_lowerCamelCase    )					,  _lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']    )					,  [25_6203, 3]    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:str 							=       tempfile.mkdtemp()
       a						:Dict 							=       self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(_lowerCamelCase    )
       a						:int 							=       NllbTokenizer.from_pretrained(_lowerCamelCase    )
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids					,  _lowerCamelCase    )




 @require_torch
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:List[str] 							=       self.tokenizer(
           self.src_text					,  text_target=self.tgt_text					,  padding=_lowerCamelCase					,  truncation=_lowerCamelCase					,  max_length=len(self.expected_src_tokens    )					,  return_tensors='''pt'''					,  )
       a						:str 							=       shift_tokens_right(
           batch['''labels''']					,  self.tokenizer.pad_token_id					,  self.tokenizer.lang_code_to_id['''ron_Latn''']    )

       self.assertIsInstance(_lowerCamelCase					,  _lowerCamelCase    )

       self.assertEqual((2, 15)					,  batch.input_ids.shape    )
       self.assertEqual((2, 15)					,  batch.attention_mask.shape    )
       a						:str 							=       batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens					,  _lowerCamelCase    )
       self.assertEqual(_lowerCamelCase					,  batch.decoder_input_ids[0, 0]    )  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens					,  [EN_CODE]    )
       self.assertEqual(self.tokenizer.suffix_tokens					,  [self.tokenizer.eos_token_id]    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Tuple 							=       self.tokenizer(self.src_text					,  padding=_lowerCamelCase					,  truncation=_lowerCamelCase					,  max_length=3					,  return_tensors='''pt'''    )
       a						:Any 							=       self.tokenizer(
           text_target=self.tgt_text					,  padding=_lowerCamelCase					,  truncation=_lowerCamelCase					,  max_length=10					,  return_tensors='''pt'''    )
       a						:List[str] 							=       targets['''input_ids''']
       a						:Union[str, Any] 							=       shift_tokens_right(
           _lowerCamelCase					,  self.tokenizer.pad_token_id					,  decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					,  )

       self.assertEqual(batch.input_ids.shape[1]					,  3    )
       self.assertEqual(batch.decoder_input_ids.shape[1]					,  10    )




 @require_torch
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:List[str] 							=       self.tokenizer._build_translation_inputs(
           '''A test'''					,  return_tensors='''pt'''					,  src_lang='''eng_Latn'''					,  tgt_lang='''fra_Latn'''    )

       self.assertEqual(
           nested_simplify(_lowerCamelCase    )					,  {
               # A, test, EOS, en_XX
               '''input_ids''': [[25_6047, 70, 7356, 2]],
               '''attention_mask''': [[1, 1, 1, 1]],
               # ar_AR
               '''forced_bos_token_id''': 25_6057,
           }					,  )




 @require_torch
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Optional[Any] 							=       True
       a						:int 							=       self.tokenizer(
           '''UN Chief says there is no military solution in Syria'''					,  src_lang='''eng_Latn'''					,  tgt_lang='''fra_Latn'''    )
       self.assertEqual(
           inputs.input_ids					,  [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]    )

       a						:Optional[int] 							=       False
       a						:Optional[Any] 							=       self.tokenizer(
           '''UN Chief says there is no military solution in Syria'''					,  src_lang='''eng_Latn'''					,  tgt_lang='''fra_Latn'''    )
       self.assertEqual(
           inputs.input_ids					,  [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]    )






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a  :     str       =  logging.get_logger(__name__)

__a  :     Optional[int]       =  {
    """google/vivit-b-16x2-kinetics400""": (
        """https://huggingface.co/google/vivit-b-16x2-kinetics400/resolve/main/config.json"""
    ),
    # See all Vivit models at https://huggingface.co/models?filter=vivit
}




class    _UpperCamelCase      (						_UpperCAmelCase ):


			"""simple docstring"""






			__a  : Tuple			 =						'''vivit'''





			def __init__(   self      ,   lowerCAmelCase__=2_24      ,   lowerCAmelCase__=32      ,   lowerCAmelCase__=[2, 16, 16]      ,   lowerCAmelCase__=3      ,   lowerCAmelCase__=7_68      ,   lowerCAmelCase__=12      ,   lowerCAmelCase__=12      ,   lowerCAmelCase__=30_72      ,   lowerCAmelCase__="gelu_fast"      ,   lowerCAmelCase__=0.0      ,   lowerCAmelCase__=0.0      ,   lowerCAmelCase__=0.02      ,   lowerCAmelCase__=1E-06      ,   lowerCAmelCase__=True      ,   **lowerCAmelCase__      ,   )     ->					int:

							'''simple docstring'''


							__lowercase 			=							hidden_size
							__lowercase 			=							num_hidden_layers
							__lowercase 			=							num_attention_heads
							__lowercase 			=							intermediate_size
							__lowercase 			=							hidden_act
							__lowercase 			=							hidden_dropout_prob
							__lowercase 			=							attention_probs_dropout_prob
							__lowercase 			=							initializer_range
							__lowercase 			=							layer_norm_eps

							__lowercase 			=							image_size
							__lowercase 			=							num_frames
							__lowercase 			=							tubelet_size
							__lowercase 			=							num_channels
							__lowercase 			=							qkv_bias

							super().__init__(**lowerCAmelCase__						)