import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import OPTConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
      import os

      # The slow tests are often failing with OOM error on GPU
      # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
      # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
      UpperCamelCase													=		'''platform'''

      import jax
      import jax.numpy as jnp

      from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer

def     lowercase_       (    _lowerCamelCase							: Optional[int]	,			_lowerCamelCase							: Optional[Any]	,			_lowerCamelCase							: str=None	,			_lowerCamelCase							: Tuple=None):
    if attention_mask is None:
        lowercase__				:     Optional[int]					      =       np.where(input_ids != config.pad_token_id	,			1	,			0)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
    }



@require_flax
class 						snake_case_ :






   def __init__(   self       :  Optional[Any]			,		lowercase_       :  Tuple			,		lowercase_       :  Union[str, Any]=13			,		lowercase_       :  Dict=7			,		lowercase_       :  str=True			,		lowercase_       :  Optional[int]=False			,		lowercase_       :  List[str]=99			,		lowercase_       :  Optional[Any]=16			,		lowercase_       :  Optional[Any]=2			,		lowercase_       :  int=4			,		lowercase_       :  List[Any]=4			,		lowercase_       :  Dict="gelu"			,		lowercase_       :  str=0.1			,		lowercase_       :  Optional[int]=0.1			,		lowercase_       :  Optional[int]=20			,		lowercase_       :  Union[str, Any]=2			,		lowercase_       :  Dict=1			,		lowercase_       :  List[str]=0			,		lowercase_       :  Dict=16			,		lowercase_       :  List[Any]=16			,		lowercase_       :  Any=0.02			,		)      ->		Dict:
       lowercase__				:     Optional[Any]					      =       parent
       lowercase__				:     Optional[Any]					      =       batch_size
       lowercase__				:     List[Any]					      =       seq_length
       lowercase__				:     Any					      =       is_training
       lowercase__				:     Any					      =       use_labels
       lowercase__				:     int					      =       vocab_size
       lowercase__				:     int					      =       hidden_size
       lowercase__				:     Union[str, Any]					      =       num_hidden_layers
       lowercase__				:     Tuple					      =       num_attention_heads
       lowercase__				:     int					      =       intermediate_size
       lowercase__				:     List[str]					      =       hidden_act
       lowercase__				:     Dict					      =       hidden_dropout_prob
       lowercase__				:     Any					      =       attention_probs_dropout_prob
       lowercase__				:     List[str]					      =       max_position_embeddings
       lowercase__				:     Dict					      =       eos_token_id
       lowercase__				:     Tuple					      =       pad_token_id
       lowercase__				:     List[str]					      =       bos_token_id
       lowercase__				:     Any					      =       embed_dim
       lowercase__				:     Any					      =       word_embed_proj_dim
       lowercase__				:     Tuple					      =       initializer_range
       lowercase__				:     Tuple					      =       False






   def    __UpperCamelCase  (   self       :  Any						)      ->		str:
       lowercase__				:     List[Any]					      =       np.clip(ids_tensor([self.batch_size, self.seq_length - 1]			,		self.vocab_size						)			,		3			,		self.vocab_size						)
       lowercase__				:     str					      =       np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)			,		dtype=np.intaa						))			,		-1						)

       lowercase__				:     Optional[Any]					      =       OPTConfig(
           vocab_size=self.vocab_size			,		hidden_size=self.hidden_size			,		num_hidden_layers=self.num_hidden_layers			,		num_attention_heads=self.num_attention_heads			,		ffn_dim=self.intermediate_size			,		dropout=self.hidden_dropout_prob			,		attention_dropout=self.attention_probs_dropout_prob			,		max_position_embeddings=self.max_position_embeddings			,		eos_token_id=self.eos_token_id			,		bos_token_id=self.bos_token_id			,		pad_token_id=self.pad_token_id			,		embed_dim=self.embed_dim			,		is_encoder_decoder=lowercase_			,		word_embed_proj_dim=self.word_embed_proj_dim			,		initializer_range=self.initializer_range			,		use_cache=lowercase_			,		)
       lowercase__				:     str					      =       prepare_opt_inputs_dict(lowercase_			,		lowercase_						)
       return config, inputs_dict






   def    __UpperCamelCase  (   self       :  List[Any]						)      ->		Optional[int]:
       lowercase__				:     Tuple					      =       self.prepare_config_and_inputs()
       return config, inputs_dict






   def    __UpperCamelCase  (   self       :  List[Any]			,		lowercase_       :  Dict			,		lowercase_       :  List[Any]			,		lowercase_       :  str						)      ->		Dict:
       lowercase__				:     Tuple					      =       20
       lowercase__				:     List[str]					      =       model_class_name(lowercase_						)

       lowercase__				:     Tuple					      =       inputs_dict["input_ids"]
       lowercase__				:     int					      =       inputs_dict["attention_mask"]

       lowercase__				:     List[str]					      =       model.init_cache(input_ids.shape[0]			,		lowercase_						)
       lowercase__				:     Dict					      =       jnp.ones((input_ids.shape[0], max_length)			,		dtype="i4"						)

       lowercase__				:     int					      =       jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1						)[None, :]			,		(input_ids.shape[0], input_ids.shape[-1] - 1)			,		)
       lowercase__				:     Dict					      =       model(
           input_ids[:, :-1]			,		attention_mask=lowercase_			,		past_key_values=lowercase_			,		position_ids=lowercase_			,		)

       lowercase__				:     List[Any]					      =       jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]			,		dtype="i4"						)
       lowercase__				:     List[Any]					      =       model(
           input_ids[:, -1:]			,		attention_mask=lowercase_			,		past_key_values=outputs_cache.past_key_values			,		position_ids=lowercase_			,		)

       lowercase__				:     Optional[Any]					      =       model(lowercase_						)

       lowercase__				:     Any					      =       np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
       self.parent.assertTrue(diff < 1E-3			,		msg=F'''Max diff is {diff}'''						)







   def    __UpperCamelCase  (   self       :  List[str]			,		lowercase_       :  Tuple			,		lowercase_       :  Union[str, Any]			,		lowercase_       :  Dict						)      ->		str:
       lowercase__				:     List[str]					      =       20
       lowercase__				:     int					      =       model_class_name(lowercase_						)

       lowercase__				:     Union[str, Any]					      =       (
           inputs_dict["input_ids"],
           inputs_dict["attention_mask"],
       )

       lowercase__				:     List[Any]					      =       jnp.concatenate(
           [
               attention_mask,
               jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])						),
           ]			,		axis=-1			,		)

       lowercase__				:     Optional[int]					      =       model.init_cache(input_ids.shape[0]			,		lowercase_						)
       lowercase__				:     Union[str, Any]					      =       jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1						)[None, :]			,		(input_ids.shape[0], input_ids.shape[-1] - 1)			,		)

       lowercase__				:     int					      =       model(
           input_ids[:, :-1]			,		attention_mask=lowercase_			,		past_key_values=lowercase_			,		position_ids=lowercase_			,		)
       lowercase__				:     Any					      =       jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]			,		dtype="i4"						)
       lowercase__				:     Tuple					      =       model(
           input_ids[:, -1:]			,		past_key_values=outputs_cache.past_key_values			,		attention_mask=lowercase_			,		position_ids=lowercase_			,		)

       lowercase__				:     Union[str, Any]					      =       model(lowercase_			,		attention_mask=lowercase_						)

       lowercase__				:     Optional[int]					      =       np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
       self.parent.assertTrue(diff < 1E-3			,		msg=F'''Max diff is {diff}'''						)



@require_flax
class 						snake_case_ (	__A						,unittest.TestCase						,__A ):
   __A			:	List[Any]        =       (FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
   __A			:	Optional[int]        =       () if is_flax_available() else ()






   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		Tuple:
       lowercase__				:     Optional[int]					      =       FlaxOPTModelTester(self						)






   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		List[str]:
       lowercase__				:     List[Any]					      =       self.model_tester.prepare_config_and_inputs()
       for model_class in self.all_model_classes:
           self.model_tester.check_use_cache_forward(lowercase_			,		lowercase_			,		lowercase_						)






   def    __UpperCamelCase  (   self       :  Dict						)      ->		Any:
       lowercase__				:     List[str]					      =       self.model_tester.prepare_config_and_inputs()
       for model_class in self.all_model_classes:
           self.model_tester.check_use_cache_forward_with_attn_mask(lowercase_			,		lowercase_			,		lowercase_						)







   @slow
   def    __UpperCamelCase  (   self       :  List[Any]						)      ->		str:
       for model_class_name in self.all_model_classes:
           lowercase__				:     Any					      =       model_class_name.from_pretrained("facebook/opt-125m"						)
           lowercase__				:     List[Any]					      =       np.ones((1, 1)						) * model.config.eos_token_id
           lowercase__				:     Optional[int]					      =       model(lowercase_						)
           self.assertIsNotNone(lowercase_						)



@require_sentencepiece
@require_flax
class 						snake_case_ (	unittest.TestCase ):






   @slow
   def    __UpperCamelCase  (   self       :  str						)      ->		Any:
       lowercase__				:     Tuple					      =       FlaxOPTModel.from_pretrained("facebook/opt-350m"						)
       lowercase__				:     Any					      =       jnp.array([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]						)
       lowercase__				:     Optional[int]					      =       model(input_ids=lowercase_						).last_hidden_state
       lowercase__				:     List[Any]					      =       (1, 11, 5_12)
       self.assertEqual(output.shape			,		lowercase_						)
       lowercase__				:     Tuple					      =       jnp.array(
           [[-0.28_67, -1.92_56, -0.30_62], [-1.27_11, -0.13_37, -0.18_97], [0.41_09, 0.11_87, -1.31_42]]						)
       self.assertTrue(jnp.allclose(output[:, :3, :3]			,		lowercase_			,		atol=4E-2						)						)



@require_flax
@slow
class 						snake_case_ (	unittest.TestCase ):






   def    __UpperCamelCase  (   self       :  Tuple						)      ->		Optional[int]:
       super().setUp()
       lowercase__				:     int					      =       "facebook/opt-350m"







   def    __UpperCamelCase  (   self       :  List[Any]						)      ->		Union[str, Any]:
       lowercase__				:     Any					      =       FlaxOPTForCausalLM.from_pretrained(self.path_model						)
       lowercase__				:     Optional[Any]					      =       GPTaTokenizer.from_pretrained(self.path_model						)

       lowercase__				:     Any					      =       [
           "Today is a beautiful day and I want to",
           "In the city of",
           "Paris is the capital of France and",
           "Computers and mobile phones have taken",
       ]
       # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
       lowercase__				:     List[str]					      =       tokenizer(lowercase_			,		return_tensors="jax"			,		padding=lowercase_			,		add_special_tokens=lowercase_						)
       lowercase__				:     Dict					      =       model(inputs.input_ids			,		attention_mask=inputs.attention_mask						)[0].mean(axis=-1						)
       lowercase__				:     Dict					      =       jnp.array(
           [
               [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
               [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
               [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
               [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
           ]						)
       self.assertTrue(jnp.allclose(lowercase_			,		lowercase_			,		atol=4E-2						)						)

       lowercase__				:     Tuple					      =       jax.jit(lowercase_						)
       lowercase__				:     Dict					      =       model(inputs.input_ids			,		attention_mask=inputs.attention_mask						)[0].mean(axis=-1						)
       self.assertTrue(jnp.allclose(lowercase_			,		lowercase_			,		atol=4E-2						)						)



@require_flax
@slow
class 						snake_case_ (	unittest.TestCase ):






   @property
   def    __UpperCamelCase  (   self       :  List[str]						)      ->		str:
       return [
           "Today is a beautiful day and I want",
           "In the city of",
           "Paris is the capital of France and",
           "Computers and mobile phones have taken",
       ]






   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		Any:
       lowercase__				:     Optional[int]					      =       "facebook/opt-125m"

       lowercase__				:     Union[str, Any]					      =       [
           "Today is a beautiful day and I want to",
           "In the city of New York, the city",
           "Paris is the capital of France and the capital",
           "Computers and mobile phones have taken over the",
       ]

       lowercase__				:     Optional[int]					      =       []

       lowercase__				:     Optional[int]					      =       FlaxOPTForCausalLM.from_pretrained(lowercase_						)
       lowercase__				:     Any					      =       GPTaTokenizer.from_pretrained(lowercase_						)

       for prompt in self.prompts:
           lowercase__				:     List[str]					      =       tokenizer(lowercase_			,		return_tensors="jax"						).input_ids

           lowercase__				:     List[Any]					      =       model.generate(lowercase_			,		max_length=10						)
           lowercase__				:     List[Any]					      =       generated_ids[0]

           lowercase__				:     Union[str, Any]					      =       tokenizer.batch_decode(lowercase_			,		skip_special_tokens=lowercase_						)
           predicted_outputs += generated_string

       self.assertListEqual(lowercase_			,		lowercase_						)






   def    __UpperCamelCase  (   self       :  List[str]						)      ->		Dict:
       lowercase__				:     Tuple					      =       "facebook/opt-350m"

       lowercase__				:     List[Any]					      =       [
           "Today is a beautiful day and I want to",
           "In the city of San Francisco, the city",
           "Paris is the capital of France and the capital",
           "Computers and mobile phones have taken over the",
       ]

       lowercase__				:     int					      =       []
       lowercase__				:     Dict					      =       FlaxOPTForCausalLM.from_pretrained(lowercase_						)
       lowercase__				:     int					      =       GPTaTokenizer.from_pretrained(lowercase_						)

       for prompt in self.prompts:
           lowercase__				:     Optional[Any]					      =       tokenizer(lowercase_			,		return_tensors="jax"						).input_ids

           lowercase__				:     List[str]					      =       model.generate(lowercase_			,		max_length=10						)
           lowercase__				:     Optional[int]					      =       generated_ids[0]

           lowercase__				:     Dict					      =       tokenizer.batch_decode(lowercase_			,		skip_special_tokens=lowercase_						)
           predicted_outputs += generated_string

       self.assertListEqual(lowercase_			,		lowercase_						)






   def    __UpperCamelCase  (   self       :  Optional[int]						)      ->		Any:
       lowercase__				:     Any					      =       "facebook/opt-125m"
       lowercase__				:     Dict					      =       [
           "Today is a beautiful day and I want to thank",
           "In the city of Rome Canaver Canaver Canaver Canaver",
       ]
       lowercase__				:     Union[str, Any]					      =       FlaxOPTForCausalLM.from_pretrained(lowercase_						)
       lowercase__				:     Dict					      =       GPTaTokenizer.from_pretrained(lowercase_						)
       lowercase__				:     int					      =       tokenizer(
           [
               "Today is a beautiful day and I want to",
               "In the city of",
           ]			,		return_tensors="jax"			,		padding=lowercase_			,		)

       lowercase__				:     int					      =       jax.jit(model.generate						)

       lowercase__				:     Optional[Any]					      =       jit_generate(inputs["input_ids"]			,		attention_mask=inputs["attention_mask"]						).sequences

       lowercase__				:     Optional[Any]					      =       tokenizer.batch_decode(lowercase_			,		skip_special_tokens=lowercase_						)

       self.assertIsNotNone(lowercase_			,		lowercase_						)







   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		Tuple:
       lowercase__				:     Tuple					      =       "facebook/opt-350m"

       lowercase__				:     List[str]					      =       GPTaTokenizer.from_pretrained(lowercase_						)
       lowercase__				:     List[Any]					      =       FlaxOPTForCausalLM.from_pretrained(lowercase_						)

       lowercase__				:     Optional[int]					      =       "left"

       # use different length sentences to test batching
       lowercase__				:     Optional[Any]					      =       [
           "Hello, my dog is a little",
           "Today, I",
       ]

       lowercase__				:     Any					      =       tokenizer(lowercase_			,		return_tensors="jax"			,		padding=lowercase_						)
       lowercase__				:     List[str]					      =       inputs["input_ids"]

       lowercase__				:     Optional[Any]					      =       model.generate(input_ids=lowercase_			,		attention_mask=inputs["attention_mask"]			,		trace=lowercase_						)

       lowercase__				:     Tuple					      =       tokenizer(sentences[0]			,		return_tensors="jax"						).input_ids
       lowercase__				:     Any					      =       model.generate(input_ids=lowercase_						)

       lowercase__				:     Dict					      =       inputs_non_padded.shape[-1] - inputs["attention_mask"][-1].sum()
       lowercase__				:     Union[str, Any]					      =       tokenizer(sentences[1]			,		return_tensors="jax"						).input_ids
       lowercase__				:     List[str]					      =       model.generate(input_ids=lowercase_			,		max_length=model.config.max_length - num_paddings						)

       lowercase__				:     Union[str, Any]					      =       tokenizer.batch_decode(outputs[0]			,		skip_special_tokens=lowercase_						)
       lowercase__				:     str					      =       tokenizer.decode(output_non_padded[0][0]			,		skip_special_tokens=lowercase_						)
       lowercase__				:     Optional[int]					      =       tokenizer.decode(output_padded[0][0]			,		skip_special_tokens=lowercase_						)

       lowercase__				:     Optional[int]					      =       [
           "Hello, my dog is a little bit of a dork.\nI'm a little bit",
           "Today, I was in the middle of a conversation with a friend about the",
       ]
       self.assertListEqual(lowercase_			,		lowercase_						)
       self.assertListEqual(lowercase_			,		[non_padded_sentence, padded_sentence]						)




import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer



class 						snake_case_ (	tf.keras.layers.Layer ):






   def __init__(   self       :  Optional[int]			,		lowercase_       :  List			,		lowercase_       :  bool			,		lowercase_       :  int = None			,		lowercase_       :  int = None			,		lowercase_       :  int = None			,		lowercase_       :  str = "longest"			,		lowercase_       :  bool = True			,		lowercase_       :  int = 5_12			,		lowercase_       :  int = None			,		lowercase_       :  bool = True			,		lowercase_       :  bool = True			,		lowercase_       :  bool = True			,		**lowercase_       :  Tuple			,		)      ->		Any:
       super().__init__()
       if use_fast_bert_tokenizer:
           lowercase__				:     Optional[Any]					      =       FastBertTokenizer(
               lowercase_			,		token_out_type=tf.intaa			,		lower_case_nfd_strip_accents=lowercase_			,		**lowercase_						)
       else:
           lowercase__				:     List[str]					      =       tf.lookup.StaticVocabularyTable(
               tf.lookup.KeyValueTensorInitializer(
                   keys=lowercase_			,		key_dtype=tf.string			,		values=tf.range(tf.size(lowercase_			,		out_type=tf.intaa						)			,		dtype=tf.intaa						)			,		value_dtype=tf.intaa			,		)			,		num_oov_buckets=1			,		)
           lowercase__				:     Optional[Any]					      =       BertTokenizerLayer(
               lowercase_			,		token_out_type=tf.intaa			,		lower_case=lowercase_			,		**lowercase_						)

       lowercase__				:     Optional[int]					      =       vocab_list
       lowercase__				:     List[Any]					      =       do_lower_case
       lowercase__				:     Dict					      =       cls_token_id or vocab_list.index("[CLS]"						)
       lowercase__				:     Any					      =       sep_token_id or vocab_list.index("[SEP]"						)
       lowercase__				:     Union[str, Any]					      =       pad_token_id or vocab_list.index("[PAD]"						)
       lowercase__				:     List[str]					      =       ShrinkLongestTrimmer(max_length - 3			,		axis=1						)  # Allow room for special tokens
       lowercase__				:     Dict					      =       max_length
       lowercase__				:     List[str]					      =       padding
       lowercase__				:     Tuple					      =       truncation
       lowercase__				:     Tuple					      =       pad_to_multiple_of
       lowercase__				:     Optional[Any]					      =       return_token_type_ids
       lowercase__				:     Dict					      =       return_attention_mask






   @classmethod
   def    __UpperCamelCase  (   cls       :  Union[str, Any]			,		lowercase_       :  "PreTrainedTokenizerBase"			,		**lowercase_       :  Optional[Any]						)      ->		int:  # noqa: F821
       lowercase__				:     Optional[Any]					      =       kwargs.pop("do_lower_case"			,		lowercase_						)
       lowercase__				:     Optional[Any]					      =       tokenizer.do_lower_case if do_lower_case is None else do_lower_case
       lowercase__				:     Union[str, Any]					      =       kwargs.pop("cls_token_id"			,		lowercase_						)
       lowercase__				:     List[Any]					      =       tokenizer.cls_token_id if cls_token_id is None else cls_token_id
       lowercase__				:     Dict					      =       kwargs.pop("sep_token_id"			,		lowercase_						)
       lowercase__				:     Dict					      =       tokenizer.sep_token_id if sep_token_id is None else sep_token_id
       lowercase__				:     Any					      =       kwargs.pop("pad_token_id"			,		lowercase_						)
       lowercase__				:     Tuple					      =       tokenizer.pad_token_id if pad_token_id is None else pad_token_id

       lowercase__				:     Union[str, Any]					      =       tokenizer.get_vocab()
       lowercase__				:     Tuple					      =       sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,		key=lambda lowercase_						: x[1]						)
       lowercase__				:     Tuple					      =       [entry[0] for entry in vocab]
       return cls(
           vocab_list=lowercase_			,		do_lower_case=lowercase_			,		cls_token_id=lowercase_			,		sep_token_id=lowercase_			,		pad_token_id=lowercase_			,		**lowercase_			,		)






   @classmethod
   def    __UpperCamelCase  (   cls       :  Any			,		lowercase_       :  Union[str, os.PathLike]			,		*lowercase_       :  Optional[Any]			,		**lowercase_       :  List[Any]						)      ->		Optional[int]:
       try:
           lowercase__				:     str					      =       BertTokenizer.from_pretrained(lowercase_			,		*lowercase_			,		**lowercase_						)
       except:  # noqa: E722
           from .tokenization_bert_fast import BertTokenizerFast

           lowercase__				:     int					      =       BertTokenizerFast.from_pretrained(lowercase_			,		*lowercase_			,		**lowercase_						)
       return cls.from_tokenizer(lowercase_			,		**lowercase_						)






   def    __UpperCamelCase  (   self       :  List[str]			,		lowercase_       :  List[Any]						)      ->		Any:
       if self.do_lower_case:
           lowercase__				:     Optional[int]					      =       case_fold_utfa(lowercase_						)
       lowercase__				:     Optional[Any]					      =       self.tf_tokenizer.tokenize(lowercase_						)
       return tokens.merge_dims(1			,		-1						)






   def    __UpperCamelCase  (   self       :  List[str]			,		lowercase_       :  Optional[int]			,		lowercase_       :  Dict=None			,		lowercase_       :  Optional[int]=None			,		lowercase_       :  Optional[int]=None			,		lowercase_       :  Tuple=None			,		lowercase_       :  Union[str, Any]=None			,		lowercase_       :  Optional[Any]=None			,		lowercase_       :  List[Any]=None			,		)      ->		int:
       if padding is None:
           lowercase__				:     str					      =       self.padding
       if padding not in ("longest", "max_length"):
           raise ValueError("Padding must be either 'longest' or 'max_length'!"						)
       if max_length is not None and text_pair is not None:
           # Because we have to instantiate a Trimmer to do it properly
           raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"						)
       if max_length is None:
           lowercase__				:     int					      =       self.max_length
       if truncation is None:
           lowercase__				:     int					      =       self.truncation
       if pad_to_multiple_of is None:
           lowercase__				:     Union[str, Any]					      =       self.pad_to_multiple_of
       if return_token_type_ids is None:
           lowercase__				:     Dict					      =       self.return_token_type_ids
       if return_attention_mask is None:
           lowercase__				:     Dict					      =       self.return_attention_mask
       if not isinstance(lowercase_			,		tf.Tensor						):
           lowercase__				:     Union[str, Any]					      =       tf.convert_to_tensor(lowercase_						)
       if text_pair is not None and not isinstance(lowercase_			,		tf.Tensor						):
           lowercase__				:     Optional[int]					      =       tf.convert_to_tensor(lowercase_						)
       if text_pair is not None:
           if text.shape.rank > 1:
               raise ValueError("text argument should not be multidimensional when a text pair is supplied!"						)
           if text_pair.shape.rank > 1:
               raise ValueError("text_pair should not be multidimensional!"						)
       if text.shape.rank == 2:
           lowercase__      ,     lowercase__				:     List[Any]					      =       text[:, 0], text[:, 1]
       lowercase__				:     List[Any]					      =       self.unpaired_tokenize(lowercase_						)
       if text_pair is None:  # Unpaired text
           if truncation:
               lowercase__				:     Any					      =       text[:, : max_length - 2]  # Allow room for special tokens
           lowercase__      ,     lowercase__				:     Dict					      =       combine_segments(
               (text,)			,		start_of_sequence_id=self.cls_token_id			,		end_of_segment_id=self.sep_token_id						)
       else:  # Paired text
           lowercase__				:     Any					      =       self.unpaired_tokenize(lowercase_						)
           if truncation:
               lowercase__      ,     lowercase__				:     List[Any]					      =       self.paired_trimmer.trim([text, text_pair]						)
           lowercase__      ,     lowercase__				:     str					      =       combine_segments(
               (text, text_pair)			,		start_of_sequence_id=self.cls_token_id			,		end_of_segment_id=self.sep_token_id						)
       if padding == "longest":
           lowercase__				:     List[str]					      =       input_ids.bounding_shape(axis=1						)
           if pad_to_multiple_of is not None:
               # No ceiling division in tensorflow, so we negate floordiv instead
               lowercase__				:     List[Any]					      =       pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,		lowercase_						))
       else:
           lowercase__				:     Optional[Any]					      =       max_length

       lowercase__      ,     lowercase__				:     str					      =       pad_model_inputs(lowercase_			,		max_seq_length=lowercase_			,		pad_value=self.pad_token_id						)
       lowercase__				:     str					      =       {"input_ids": input_ids}
       if return_attention_mask:
           lowercase__				:     List[Any]					      =       attention_mask
       if return_token_type_ids:
           lowercase__      ,     lowercase__				:     List[str]					      =       pad_model_inputs(
               lowercase_			,		max_seq_length=lowercase_			,		pad_value=self.pad_token_id						)
           lowercase__				:     List[Any]					      =       token_type_ids
       return output







   def    __UpperCamelCase  (   self       :  Any						)      ->		Optional[Any]:
       return {
           "vocab_list": self.vocab_list,
           "do_lower_case": self.do_lower_case,
           "cls_token_id": self.cls_token_id,
           "sep_token_id": self.sep_token_id,
           "pad_token_id": self.pad_token_id,
       }



