





'''simple docstring'''







import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_ta import TaTokenizer
else:
					lowerCAmelCase_					:    Optional[Any]      	=			None


lowerCAmelCase_					:    List[Any]      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    Any      	=			{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_					:    int      	=			{
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
lowerCAmelCase_					:    Tuple      	=			{
    't5-small': 5_12,
    't5-base': 5_12,
    't5-large': 5_12,
    't5-3b': 5_12,
    't5-11b': 5_12,
}
class 				__SCREAMING_SNAKE_CASE  (snake_case_      ):






				"""simple docstring"""







				__a            =VOCAB_FILES_NAMES
				__a            =PRETRAINED_VOCAB_FILES_MAP
				__a            =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a            =['''input_ids''', '''attention_mask''']
				__a            =TaTokenizer

				__a            =[]




				def __init__(  self     :					Optional[int]					,				__a     :					Union[str, Any]=None					,				__a     :					Union[str, Any]=None					,				__a     :					Tuple="</s>"					,				__a     :					int="<unk>"					,				__a     :					str="<pad>"					,				__a     :					Union[str, Any]=1_00					,				__a     :					Optional[int]=None					,				**__a     :					Dict					,				):




								if extra_ids > 0 and additional_special_tokens is None:
												_a								= [f'<extra_id_{i}>' for i in range(UpperCamelCase__					)]
								elif extra_ids > 0 and additional_special_tokens is not None:
												# Check that we have the right number of extra special tokens
												_a								= len(set(filter(lambda __a					: bool("extra_id_" in str(UpperCamelCase__					)					)					,				UpperCamelCase__					)					)					)
												if extra_tokens != extra_ids:
																raise ValueError(
																    f'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
																    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
																    " tokens"					)

								super().__init__(
								    UpperCamelCase__					,				tokenizer_file=UpperCamelCase__					,				eos_token=UpperCamelCase__					,				unk_token=UpperCamelCase__					,				pad_token=UpperCamelCase__					,				extra_ids=UpperCamelCase__					,				additional_special_tokens=UpperCamelCase__					,				**UpperCamelCase__					,				)

								_a								= vocab_file
								_a								= False if not self.vocab_file else True
								_a								= extra_ids




				@staticmethod
				def   UpperCamelCase__   (  __a     :					Union[str, Any]					,				__a     :					Optional[Any]					,				__a     :					Tuple					):




								if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
												_a								= TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
												if init_max_model_length is not None and init_max_model_length != max_model_length:
																return init_max_model_length
												elif init_max_model_length is None:
																warnings.warn(
																    "This tokenizer was incorrectly instantiated with a model max length of"
																    f' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'
																    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
																    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
																    f' {pretrained_model_name_or_path} automatically truncating your input to'
																    f' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'
																    f' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'
																    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
																    " instantiate this tokenizer with `model_max_length` set to your preferred value."					,				UpperCamelCase__					,				)

								return max_model_length




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				__a     :					Union[str, Any]					,				__a     :					str = None					):




								if not self.can_save_slow_tokenizer:
												raise ValueError(
												    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
												    "tokenizer."					)

								if not os.path.isdir(UpperCamelCase__					):
												logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
												return
								_a								= os.path.join(
								    UpperCamelCase__					,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)

								if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase__					):
												copyfile(self.vocab_file					,				UpperCamelCase__					)
												logger.info(f'Copy vocab file to {out_vocab_file}'					)

								return (out_vocab_file,)




				def   UpperCamelCase__   (  self     :					int					,				__a     :					Union[str, Any]					,				__a     :					int = None					):
								_a								= token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
												return self.prefix_tokens + token_ids_a
								else:
												_a								= token_ids_a + [self.eos_token_id]
												return self.prefix_tokens + token_ids_a + token_ids_a




				def   UpperCamelCase__   (  self     :					Tuple					,				__a     :					Union[str, Any]					,				__a     :					Any = None					):
								_a								= [self.eos_token_id]

								if token_ids_a is None:
												return len(token_ids_a + eos					) * [0]
								return len(token_ids_a + eos + token_ids_a + eos					) * [0]




				def   UpperCamelCase__   (  self     :					List[str]					):
								return list(
								    set(filter(lambda __a					: bool(re.search(r"<extra_id_\d+>"					,				UpperCamelCase__					)					) is not None					,				self.additional_special_tokens					)					)					)




				def   UpperCamelCase__   (  self     :					List[str]					):
								return [self.convert_tokens_to_ids(UpperCamelCase__					) for token in self.get_sentinel_tokens()]




"""simple docstring"""



from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
  import tensorflow as tf

  from transformers import TFBlipTextModel
  from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST







class _lowerCAmelCase				:







   def __init__(	self						,	UpperCamelCase__						,	UpperCamelCase__=12						,	UpperCamelCase__=7						,	UpperCamelCase__=True						,	UpperCamelCase__=True						,	UpperCamelCase__=True						,	UpperCamelCase__=99						,	UpperCamelCase__=32						,	UpperCamelCase__=32						,	UpperCamelCase__=2						,	UpperCamelCase__=4						,	UpperCamelCase__=37						,	UpperCamelCase__=0.1						,	UpperCamelCase__=0.1						,	UpperCamelCase__=512						,	UpperCamelCase__=0.02						,	UpperCamelCase__=0						,	UpperCamelCase__=None						,	)	->		Dict:

       '''simple docstring'''




       snake_case    :      Optional[Any]      					=    parent
       snake_case    :      Dict      					=    batch_size
       snake_case    :      List[str]      					=    seq_length
       snake_case    :      Dict      					=    is_training
       snake_case    :      Optional[Any]      					=    use_input_mask
       snake_case    :      Optional[int]      					=    use_labels
       snake_case    :      Tuple      					=    vocab_size
       snake_case    :      Optional[Any]      					=    hidden_size
       snake_case    :      Optional[Any]      					=    projection_dim
       snake_case    :      List[Any]      					=    num_hidden_layers
       snake_case    :      List[Any]      					=    num_attention_heads
       snake_case    :      int      					=    intermediate_size
       snake_case    :      str      					=    dropout
       snake_case    :      List[Any]      					=    attention_dropout
       snake_case    :      Any      					=    max_position_embeddings
       snake_case    :      List[Any]      					=    initializer_range
       snake_case    :      Any      					=    scope
       snake_case    :      Union[str, Any]      					=    bos_token_id







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      str      					=    ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size						)

       snake_case    :      int      					=    None
       if self.use_input_mask:
           snake_case    :      Optional[int]      					=    random_attention_mask([self.batch_size, self.seq_length]						)

       if input_mask is not None:
           snake_case    :      Tuple      					=    input_mask.numpy()
           snake_case					,snake_case    :      str      					=    input_mask.shape
           snake_case    :      Tuple      					=    np.random.randint(1						,	seq_length - 1						,	size=(batch_size,)						)
           for batch_idx, start_index in enumerate(UpperCamelCase__						):
               snake_case    :      int      					=    1
               snake_case    :      Tuple      					=    0

       snake_case    :      Union[str, Any]      					=    self.get_config()

       return config, input_ids, tf.convert_to_tensor(UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		Union[str, Any]:

       '''simple docstring'''




       return BlipTextConfig(
           vocab_size=self.vocab_size						,	hidden_size=self.hidden_size						,	projection_dim=self.projection_dim						,	num_hidden_layers=self.num_hidden_layers						,	num_attention_heads=self.num_attention_heads						,	intermediate_size=self.intermediate_size						,	dropout=self.dropout						,	attention_dropout=self.attention_dropout						,	max_position_embeddings=self.max_position_embeddings						,	initializer_range=self.initializer_range						,	bos_token_id=self.bos_token_id						,	)







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						)	->		Tuple:

       '''simple docstring'''




       snake_case    :      str      					=    TFBlipTextModel(config=UpperCamelCase__						)
       snake_case    :      Any      					=    model(UpperCamelCase__						,	attention_mask=UpperCamelCase__						,	training=UpperCamelCase__						)
       snake_case    :      Optional[int]      					=    model(UpperCamelCase__						,	training=UpperCamelCase__						)
       self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape						,	(self.batch_size, self.hidden_size)						)







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Tuple      					=    self.prepare_config_and_inputs()
       snake_case					,snake_case					,snake_case    :      Tuple      					=    config_and_inputs
       snake_case    :      str      					=    {"input_ids": input_ids, "attention_mask": input_mask}
       return config, inputs_dict







@require_tf
class _lowerCAmelCase				(      snake_case_    ,							unittest.TestCase	):
   __UpperCAmelCase :       Any							  =  (TFBlipTextModel,) if is_tf_available() else ()
   __UpperCAmelCase :       Any							  =  False
   __UpperCAmelCase :       Dict							  =  False
   __UpperCAmelCase :       List[Any]							  =  False







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      List[Any]      					=    BlipTextModelTester(self						)
       snake_case    :      Optional[int]      					=    ConfigTester(self						,	config_class=UpperCamelCase__						,	hidden_size=37						)







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       self.config_tester.run_common_tests()







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      List[str]      					=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		Union[str, Any]:

       '''simple docstring'''




       pass







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       pass







   @unittest.skip(reason="Blip does not use inputs_embeds"						)
   def  lowerCamelCase		(	self						)	->		Tuple:

       '''simple docstring'''




       pass







   @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING"						)
   def  lowerCamelCase		(	self						)	->		Optional[Any]:

       '''simple docstring'''




       pass







   @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING"						)
   def  lowerCamelCase		(	self						)	->		Any:

       '''simple docstring'''




       pass







   @slow
   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           snake_case    :      List[str]      					=    TFBlipTextModel.from_pretrained(UpperCamelCase__						)
           self.assertIsNotNone(UpperCamelCase__						)







   def  lowerCamelCase		(	self						,	UpperCamelCase__=True						)	->		Optional[int]:

       '''simple docstring'''




       super().test_pt_tf_model_equivalence(allow_missing_keys=UpperCamelCase__						)





