


"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_pegasus import PegasusTokenizer
else:
  __snake_case   			=    None


__snake_case   			=    logging.get_logger(__name__)


__snake_case   			=    """▁"""

__snake_case   			=    {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__snake_case   			=    {
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""},
    """tokenizer_file""": {
        """google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"""
    },
}

__snake_case   			=    {
    """google/pegasus-xsum""": 512,
}







class _lowerCAmelCase				(      snake_case_	):
   __UpperCAmelCase :       Union[str, Any]							  =  VOCAB_FILES_NAMES
   __UpperCAmelCase :       List[str]							  =  PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase :       List[Any]							  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase :       str							  =  PegasusTokenizer
   __UpperCAmelCase :       Union[str, Any]							  =  ['''input_ids''', '''attention_mask''']







   def __init__(	self						,	UpperCamelCase__=None						,	UpperCamelCase__=None						,	UpperCamelCase__="<pad>"						,	UpperCamelCase__="</s>"						,	UpperCamelCase__="<unk>"						,	UpperCamelCase__="<mask_2>"						,	UpperCamelCase__="<mask_1>"						,	UpperCamelCase__=None						,	UpperCamelCase__=103						,	**UpperCamelCase__						,	)	->		Any:

       '''simple docstring'''




       snake_case    :      str      					=    offset

       if additional_special_tokens is not None:
           if not isinstance(UpperCamelCase__						,	UpperCamelCase__						):
               raise TypeError(
                   F'additional_special_tokens should be of type {type(UpperCamelCase__						)}, but is'
                   F' {type(UpperCamelCase__						)}'						)

           snake_case    :      int      					=    (
               ([mask_token_sent] + additional_special_tokens)
               if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
               else additional_special_tokens
           )
           # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
           additional_special_tokens_extended += [
               F'<unk_{i}>' for i in range(len(UpperCamelCase__						)						,	self.offset - 1						)
           ]

           if len(set(UpperCamelCase__						)						) != len(UpperCamelCase__						):
               raise ValueError(
                   "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                   F' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'						)
           snake_case    :      Optional[int]      					=    additional_special_tokens_extended
       else:
           snake_case    :      Optional[int]      					=    [mask_token_sent] if mask_token_sent is not None else []
           additional_special_tokens += [F'<unk_{i}>' for i in range(2						,	self.offset						)]

       super().__init__(
           UpperCamelCase__						,	tokenizer_file=UpperCamelCase__						,	pad_token=UpperCamelCase__						,	eos_token=UpperCamelCase__						,	unk_token=UpperCamelCase__						,	mask_token=UpperCamelCase__						,	mask_token_sent=UpperCamelCase__						,	offset=UpperCamelCase__						,	additional_special_tokens=UpperCamelCase__						,	**UpperCamelCase__						,	)
       snake_case    :      Tuple      					=    vocab_file
       snake_case    :      List[str]      					=    False if not self.vocab_file else True







   def  lowerCamelCase		(	self						,	UpperCamelCase__						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Optional[int]      					=    set(self.all_special_ids						)  # call it once instead of inside list comp
       all_special_ids.remove(self.unk_token_id						)  # <unk> is only sometimes special

       if all_special_ids != set(range(len(self.additional_special_tokens						) + 3						)						):
           raise ValueError(
               "There should be 3 special tokens: mask_token, pad_token, and eos_token +"
               F' {len(self.additional_special_tokens						)} additional_special_tokens, but got {all_special_ids}'						)

       return [1 if x in all_special_ids else 0 for x in seq]







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__ = None						,	UpperCamelCase__ = False						)	->		List[int]:

       '''simple docstring'''




       if already_has_special_tokens:
           return self._special_token_mask(UpperCamelCase__						)
       elif token_ids_a is None:
           return self._special_token_mask(UpperCamelCase__						) + [1]
       else:
           return self._special_token_mask(token_ids_a + token_ids_a						) + [1]







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__=None						)	->		List[int]:

       '''simple docstring'''




       if token_ids_a is None:
           return token_ids_a + [self.eos_token_id]
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return token_ids_a + token_ids_a + [self.eos_token_id]







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__ = None						)	->		Tuple[str]:

       '''simple docstring'''




       if not self.can_save_slow_tokenizer:
           raise ValueError(
               "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
               "tokenizer."						)

       if not os.path.isdir(UpperCamelCase__						):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
           return
       snake_case    :      List[Any]      					=    os.path.join(
           UpperCamelCase__						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCamelCase__						):
           copyfile(self.vocab_file						,	UpperCamelCase__						)

       return (out_vocab_file,)









"""simple docstring"""



from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
  import tensorflow as tf

  from transformers import TFBlipTextModel
  from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST







class _lowerCAmelCase				:







   def __init__(	self						,	UpperCamelCase__						,	UpperCamelCase__=12						,	UpperCamelCase__=7						,	UpperCamelCase__=True						,	UpperCamelCase__=True						,	UpperCamelCase__=True						,	UpperCamelCase__=99						,	UpperCamelCase__=32						,	UpperCamelCase__=32						,	UpperCamelCase__=2						,	UpperCamelCase__=4						,	UpperCamelCase__=37						,	UpperCamelCase__=0.1						,	UpperCamelCase__=0.1						,	UpperCamelCase__=512						,	UpperCamelCase__=0.02						,	UpperCamelCase__=0						,	UpperCamelCase__=None						,	)	->		Dict:

       '''simple docstring'''




       snake_case    :      Optional[Any]      					=    parent
       snake_case    :      Dict      					=    batch_size
       snake_case    :      List[str]      					=    seq_length
       snake_case    :      Dict      					=    is_training
       snake_case    :      Optional[Any]      					=    use_input_mask
       snake_case    :      Optional[int]      					=    use_labels
       snake_case    :      Tuple      					=    vocab_size
       snake_case    :      Optional[Any]      					=    hidden_size
       snake_case    :      Optional[Any]      					=    projection_dim
       snake_case    :      List[Any]      					=    num_hidden_layers
       snake_case    :      List[Any]      					=    num_attention_heads
       snake_case    :      int      					=    intermediate_size
       snake_case    :      str      					=    dropout
       snake_case    :      List[Any]      					=    attention_dropout
       snake_case    :      Any      					=    max_position_embeddings
       snake_case    :      List[Any]      					=    initializer_range
       snake_case    :      Any      					=    scope
       snake_case    :      Union[str, Any]      					=    bos_token_id







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      str      					=    ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size						)

       snake_case    :      int      					=    None
       if self.use_input_mask:
           snake_case    :      Optional[int]      					=    random_attention_mask([self.batch_size, self.seq_length]						)

       if input_mask is not None:
           snake_case    :      Tuple      					=    input_mask.numpy()
           snake_case					,snake_case    :      str      					=    input_mask.shape
           snake_case    :      Tuple      					=    np.random.randint(1						,	seq_length - 1						,	size=(batch_size,)						)
           for batch_idx, start_index in enumerate(UpperCamelCase__						):
               snake_case    :      int      					=    1
               snake_case    :      Tuple      					=    0

       snake_case    :      Union[str, Any]      					=    self.get_config()

       return config, input_ids, tf.convert_to_tensor(UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		Union[str, Any]:

       '''simple docstring'''




       return BlipTextConfig(
           vocab_size=self.vocab_size						,	hidden_size=self.hidden_size						,	projection_dim=self.projection_dim						,	num_hidden_layers=self.num_hidden_layers						,	num_attention_heads=self.num_attention_heads						,	intermediate_size=self.intermediate_size						,	dropout=self.dropout						,	attention_dropout=self.attention_dropout						,	max_position_embeddings=self.max_position_embeddings						,	initializer_range=self.initializer_range						,	bos_token_id=self.bos_token_id						,	)







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						)	->		Tuple:

       '''simple docstring'''




       snake_case    :      str      					=    TFBlipTextModel(config=UpperCamelCase__						)
       snake_case    :      Any      					=    model(UpperCamelCase__						,	attention_mask=UpperCamelCase__						,	training=UpperCamelCase__						)
       snake_case    :      Optional[int]      					=    model(UpperCamelCase__						,	training=UpperCamelCase__						)
       self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape						,	(self.batch_size, self.hidden_size)						)







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Tuple      					=    self.prepare_config_and_inputs()
       snake_case					,snake_case					,snake_case    :      Tuple      					=    config_and_inputs
       snake_case    :      str      					=    {"input_ids": input_ids, "attention_mask": input_mask}
       return config, inputs_dict







@require_tf
class _lowerCAmelCase				(      snake_case_    ,							unittest.TestCase	):
   __UpperCAmelCase :       Any							  =  (TFBlipTextModel,) if is_tf_available() else ()
   __UpperCAmelCase :       Any							  =  False
   __UpperCAmelCase :       Dict							  =  False
   __UpperCAmelCase :       List[Any]							  =  False







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      List[Any]      					=    BlipTextModelTester(self						)
       snake_case    :      Optional[int]      					=    ConfigTester(self						,	config_class=UpperCamelCase__						,	hidden_size=37						)







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       self.config_tester.run_common_tests()







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      List[str]      					=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		Union[str, Any]:

       '''simple docstring'''




       pass







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       pass







   @unittest.skip(reason="Blip does not use inputs_embeds"						)
   def  lowerCamelCase		(	self						)	->		Tuple:

       '''simple docstring'''




       pass







   @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING"						)
   def  lowerCamelCase		(	self						)	->		Optional[Any]:

       '''simple docstring'''




       pass







   @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING"						)
   def  lowerCamelCase		(	self						)	->		Any:

       '''simple docstring'''




       pass







   @slow
   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           snake_case    :      List[str]      					=    TFBlipTextModel.from_pretrained(UpperCamelCase__						)
           self.assertIsNotNone(UpperCamelCase__						)







   def  lowerCamelCase		(	self						,	UpperCamelCase__=True						)	->		Optional[int]:

       '''simple docstring'''




       super().test_pt_tf_model_equivalence(allow_missing_keys=UpperCamelCase__						)





