code: 
  
  import unittest
  
  import numpy as np
  import timeout_decorator  # noqa
  
  from transformers import OPTConfig, is_flax_available
  from transformers.testing_utils import require_flax, require_sentencepiece, slow
  
  from ...generation.test_flax_utils import FlaxGenerationTesterMixin
  from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
  
  
  if is_flax_available():
  			import os
  
  			# The slow tests are often failing with OOM error on GPU
  			# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
  			# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
  			UpperCAmelCase_       :				List[str]		   =      """platform"""
  
  			import jax
  			import jax.numpy as jnp
  
  			from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer
  def        __SCREAMING_SNAKE_CASE    (      a__						:      Optional[Any] ,a__						:      int ,a__						:      str=None ,a__						:      Dict=None					)				->							int:
  	if attention_mask is None:
  		__A   : Dict								= np.where(input_ids != config.pad_token_id ,1 ,0					)
  	return {
  	    "input_ids": input_ids,
  	    "attention_mask": attention_mask,
  	}
  
  
  
  
  
  
  @require_flax
  class     lowerCamelCase_			:
  
  
  
  
  	def __init__(    self	:    Optional[Any]				,			__A	:    str				,			__A	:    List[Any]=13				,			__A	:    str=7				,			__A	:    List[str]=True				,			__A	:    Optional[int]=False				,			__A	:    Optional[int]=99				,			__A	:    Union[str, Any]=16				,			__A	:    List[str]=2				,			__A	:    str=4				,			__A	:    Dict=4				,			__A	:    List[Any]="gelu"				,			__A	:    Tuple=0.1				,			__A	:    Any=0.1				,			__A	:    List[Any]=20				,			__A	:    Dict=2				,			__A	:    Any=1				,			__A	:    int=0				,			__A	:    Optional[int]=16				,			__A	:    Tuple=16				,			__A	:    List[Any]=0.0_2				,			):
  		__A   : Tuple								= parent
  		__A   : int								= batch_size
  		__A   : Dict								= seq_length
  		__A   : Optional[int]								= is_training
  		__A   : List[str]								= use_labels
  		__A   : Optional[int]								= vocab_size
  		__A   : Optional[Any]								= hidden_size
  		__A   : str								= num_hidden_layers
  		__A   : List[Any]								= num_attention_heads
  		__A   : Any								= intermediate_size
  		__A   : Any								= hidden_act
  		__A   : List[Any]								= hidden_dropout_prob
  		__A   : Optional[int]								= attention_probs_dropout_prob
  		__A   : List[Any]								= max_position_embeddings
  		__A   : Tuple								= eos_token_id
  		__A   : Optional[int]								= pad_token_id
  		__A   : Tuple								= bos_token_id
  		__A   : int								= embed_dim
  		__A   : Any								= word_embed_proj_dim
  		__A   : str								= initializer_range
  		__A   : Optional[Any]								= False
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		__A   : Optional[Any]								= np.clip(ids_tensor([self.batch_size, self.seq_length - 1]				,			self.vocab_size       )				,			3				,			self.vocab_size       )
  		__A   : int								= np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)				,			dtype=np.intaa       ))				,			-1       )
  
  		__A   : Union[str, Any]								= OPTConfig(
  		    vocab_size=self.vocab_size				,			hidden_size=self.hidden_size				,			num_hidden_layers=self.num_hidden_layers				,			num_attention_heads=self.num_attention_heads				,			ffn_dim=self.intermediate_size				,			dropout=self.hidden_dropout_prob				,			attention_dropout=self.attention_probs_dropout_prob				,			max_position_embeddings=self.max_position_embeddings				,			eos_token_id=self.eos_token_id				,			bos_token_id=self.bos_token_id				,			pad_token_id=self.pad_token_id				,			embed_dim=self.embed_dim				,			is_encoder_decoder=lowerCamelCase__				,			word_embed_proj_dim=self.word_embed_proj_dim				,			initializer_range=self.initializer_range				,			use_cache=lowerCamelCase__				,			)
  		__A   : Any								= prepare_opt_inputs_dict(lowerCamelCase__				,			lowerCamelCase__       )
  		return config, inputs_dict
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int       ):
  		__A   : Optional[int]								= self.prepare_config_and_inputs()
  		return config, inputs_dict
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    str				,			__A	:    str				,			__A	:    List[Any]				,			__A	:    List[Any]       ):
  		__A   : Tuple								= 20
  		__A   : Any								= model_class_name(lowerCamelCase__       )
  
  		__A   : str								= inputs_dict["input_ids"]
  		__A   : str								= inputs_dict["attention_mask"]
  
  		__A   : List[str]								= model.init_cache(input_ids.shape[0]				,			lowerCamelCase__       )
  		__A   : Optional[int]								= jnp.ones((input_ids.shape[0], max_length)				,			dtype="""i4"""       )
  
  		__A   : Dict								= jnp.broadcast_to(
  		    jnp.arange(input_ids.shape[-1] - 1       )[None, :]				,			(input_ids.shape[0], input_ids.shape[-1] - 1)				,			)
  		__A   : List[Any]								= model(
  		    input_ids[:, :-1]				,			attention_mask=lowerCamelCase__				,			past_key_values=lowerCamelCase__				,			position_ids=lowerCamelCase__				,			)
  
  		__A   : Tuple								= jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]				,			dtype="""i4"""       )
  		__A   : Any								= model(
  		    input_ids[:, -1:]				,			attention_mask=lowerCamelCase__				,			past_key_values=outputs_cache.past_key_values				,			position_ids=lowerCamelCase__				,			)
  
  		__A   : str								= model(lowerCamelCase__       )
  
  		__A   : List[str]								= np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
  		self.parent.assertTrue(diff < 1e-3				,			msg=F"""Max diff is {diff}"""       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]				,			__A	:    Any				,			__A	:    int				,			__A	:    Optional[int]       ):
  		__A   : Optional[Any]								= 20
  		__A   : Tuple								= model_class_name(lowerCamelCase__       )
  
  		__A   : Any								= (
  		    inputs_dict["input_ids"],
  		    inputs_dict["attention_mask"],
  		)
  
  		__A   : int								= jnp.concatenate(
  		    [
  		        attention_mask,
  		        jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])       ),
  		    ]				,			axis=-1				,			)
  
  		__A   : Tuple								= model.init_cache(input_ids.shape[0]				,			lowerCamelCase__       )
  		__A   : str								= jnp.broadcast_to(
  		    jnp.arange(input_ids.shape[-1] - 1       )[None, :]				,			(input_ids.shape[0], input_ids.shape[-1] - 1)				,			)
  
  		__A   : Optional[int]								= model(
  		    input_ids[:, :-1]				,			attention_mask=lowerCamelCase__				,			past_key_values=lowerCamelCase__				,			position_ids=lowerCamelCase__				,			)
  		__A   : Tuple								= jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]				,			dtype="""i4"""       )
  		__A   : Dict								= model(
  		    input_ids[:, -1:]				,			past_key_values=outputs_cache.past_key_values				,			attention_mask=lowerCamelCase__				,			position_ids=lowerCamelCase__				,			)
  
  		__A   : int								= model(lowerCamelCase__				,			attention_mask=lowerCamelCase__       )
  
  		__A   : List[Any]								= np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])       )       )
  		self.parent.assertTrue(diff < 1e-3				,			msg=F"""Max diff is {diff}"""       )
  
  
  
  
  
  
  @require_flax
  class     lowerCamelCase_			(  _a	,			unittest.TestCase	,			_a  ):
  	_lowercase		:      str    				=							(FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
  	_lowercase		:      List[str]    				=							() if is_flax_available() else ()
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    str       ):
  		__A   : List[Any]								= FlaxOPTModelTester(self       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
  		__A   : Tuple								= self.model_tester.prepare_config_and_inputs()
  		for model_class in self.all_model_classes:
  			self.model_tester.check_use_cache_forward(lowerCamelCase__				,			lowerCamelCase__				,			lowerCamelCase__       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
  		__A   : int								= self.model_tester.prepare_config_and_inputs()
  		for model_class in self.all_model_classes:
  			self.model_tester.check_use_cache_forward_with_attn_mask(lowerCamelCase__				,			lowerCamelCase__				,			lowerCamelCase__       )
  
  
  
  
  	@slow
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		for model_class_name in self.all_model_classes:
  			__A   : Union[str, Any]								= model_class_name.from_pretrained("""facebook/opt-125m"""       )
  			__A   : int								= np.ones((1, 1)       ) * model.config.eos_token_id
  			__A   : Optional[int]								= model(lowerCamelCase__       )
  			self.assertIsNotNone(lowerCamelCase__       )
  
  
  
  
  
  
  @require_sentencepiece
  @require_flax
  class     lowerCamelCase_			(  unittest.TestCase  ):
  
  
  
  
  	@slow
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : List[Any]								= FlaxOPTModel.from_pretrained("""facebook/opt-350m"""       )
  		__A   : Tuple								= jnp.array([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]       )
  		__A   : int								= model(input_ids=lowerCamelCase__       ).last_hidden_state
  		__A   : Optional[int]								= (1, 11, 512)
  		self.assertEqual(output.shape				,			lowerCamelCase__       )
  		__A   : List[Any]								= jnp.array(
  		    [[-0.2_8_6_7, -1.9_2_5_6, -0.3_0_6_2], [-1.2_7_1_1, -0.1_3_3_7, -0.1_8_9_7], [0.4_1_0_9, 0.1_1_8_7, -1.3_1_4_2]]       )
  		self.assertTrue(jnp.allclose(output[:, :3, :3]				,			lowerCamelCase__				,			atol=4e-2       )       )
  
  
  
  
  
  
  @require_flax
  @slow
  class     lowerCamelCase_			(  unittest.TestCase  ):
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict       ):
  		super().setUp()
  		__A   : Optional[Any]								= "facebook/opt-350m"
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any       ):
  		__A   : Union[str, Any]								= FlaxOPTForCausalLM.from_pretrained(self.path_model       )
  		__A   : Dict								= GPTaTokenizer.from_pretrained(self.path_model       )
  
  		__A   : List[Any]								= [
  		    "Today is a beautiful day and I want to",
  		    "In the city of",
  		    "Paris is the capital of France and",
  		    "Computers and mobile phones have taken",
  		]
  		# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
  		__A   : List[Any]								= tokenizer(lowerCamelCase__				,			return_tensors="""jax"""				,			padding=lowerCamelCase__				,			add_special_tokens=lowerCamelCase__       )
  		__A   : List[Any]								= model(inputs.input_ids				,			attention_mask=inputs.attention_mask       )[0].mean(axis=-1       )
  		__A   : Dict								= jnp.array(
  		    [
  		        [1.3_8_5_1, -1_3.8_9_2_3, -1_0.5_2_2_9, -1_0.7_5_3_3, -0.2_3_0_9, -1_0.2_3_8_4, -0.5_3_6_5, -9.0_9_4_7, -5.1_6_7_0],
  		        [-4.7_0_7_3, -1_0.6_2_7_6, -3.9_4_1_5, -2_1.5_2_4_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2, -0.2_8_2_2],
  		        [0.6_2_4_7, -3.4_2_2_9, -8.9_1_7_9, -1.4_2_9_7, -1_4.1_6_5_0, 1.4_1_4_6, -9.0_2_1_8, -0.2_7_0_3, -0.2_7_0_3],
  		        [6.4_7_8_3, -1.9_9_1_3, -1_0.7_9_2_6, -2.3_3_3_6, 1.5_0_9_2, -0.9_9_7_4, -6.8_2_1_3, 1.3_4_7_7, 1.3_4_7_7],
  		    ]       )
  		self.assertTrue(jnp.allclose(lowerCamelCase__				,			lowerCamelCase__				,			atol=4e-2       )       )
  
  		__A   : Optional[Any]								= jax.jit(lowerCamelCase__       )
  		__A   : List[Any]								= model(inputs.input_ids				,			attention_mask=inputs.attention_mask       )[0].mean(axis=-1       )
  		self.assertTrue(jnp.allclose(lowerCamelCase__				,			lowerCamelCase__				,			atol=4e-2       )       )
  
  
  
  
  
  
  @require_flax
  @slow
  class     lowerCamelCase_			(  unittest.TestCase  ):
  
  
  
  
  	@property
  	def 					lowerCAmelCase_					(    self	:    Optional[Any]       ):
  		return [
  		    "Today is a beautiful day and I want",
  		    "In the city of",
  		    "Paris is the capital of France and",
  		    "Computers and mobile phones have taken",
  		]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int       ):
  		__A   : List[Any]								= "facebook/opt-125m"
  
  		__A   : Tuple								= [
  		    "Today is a beautiful day and I want to",
  		    "In the city of New York, the city",
  		    "Paris is the capital of France and the capital",
  		    "Computers and mobile phones have taken over the",
  		]
  
  		__A   : Tuple								= []
  
  		__A   : int								= FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__       )
  		__A   : int								= GPTaTokenizer.from_pretrained(lowerCamelCase__       )
  
  		for prompt in self.prompts:
  			__A   : Tuple								= tokenizer(lowerCamelCase__				,			return_tensors="""jax"""       ).input_ids
  
  			__A   : Optional[Any]								= model.generate(lowerCamelCase__				,			max_length=10       )
  			__A   : Optional[int]								= generated_ids[0]
  
  			__A   : List[str]								= tokenizer.batch_decode(lowerCamelCase__				,			skip_special_tokens=lowerCamelCase__       )
  			predicted_outputs += generated_string
  
  		self.assertListEqual(lowerCamelCase__				,			lowerCamelCase__       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Tuple       ):
  		__A   : Union[str, Any]								= "facebook/opt-350m"
  
  		__A   : List[str]								= [
  		    "Today is a beautiful day and I want to",
  		    "In the city of San Francisco, the city",
  		    "Paris is the capital of France and the capital",
  		    "Computers and mobile phones have taken over the",
  		]
  
  		__A   : Tuple								= []
  		__A   : List[Any]								= FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__       )
  		__A   : Any								= GPTaTokenizer.from_pretrained(lowerCamelCase__       )
  
  		for prompt in self.prompts:
  			__A   : Dict								= tokenizer(lowerCamelCase__				,			return_tensors="""jax"""       ).input_ids
  
  			__A   : Dict								= model.generate(lowerCamelCase__				,			max_length=10       )
  			__A   : str								= generated_ids[0]
  
  			__A   : str								= tokenizer.batch_decode(lowerCamelCase__				,			skip_special_tokens=lowerCamelCase__       )
  			predicted_outputs += generated_string
  
  		self.assertListEqual(lowerCamelCase__				,			lowerCamelCase__       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict       ):
  		__A   : Dict								= "facebook/opt-125m"
  		__A   : Optional[Any]								= [
  		    "Today is a beautiful day and I want to thank",
  		    "In the city of Rome Canaver Canaver Canaver Canaver",
  		]
  		__A   : int								= FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__       )
  		__A   : List[Any]								= GPTaTokenizer.from_pretrained(lowerCamelCase__       )
  		__A   : List[str]								= tokenizer(
  		    [
  		        """Today is a beautiful day and I want to""",
  		        """In the city of""",
  		    ]				,			return_tensors="""jax"""				,			padding=lowerCamelCase__				,			)
  
  		__A   : int								= jax.jit(model.generate       )
  
  		__A   : str								= jit_generate(inputs["""input_ids"""]				,			attention_mask=inputs["""attention_mask"""]       ).sequences
  
  		__A   : Optional[int]								= tokenizer.batch_decode(lowerCamelCase__				,			skip_special_tokens=lowerCamelCase__       )
  
  		self.assertIsNotNone(lowerCamelCase__				,			lowerCamelCase__       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Tuple								= "facebook/opt-350m"
  
  		__A   : List[Any]								= GPTaTokenizer.from_pretrained(lowerCamelCase__       )
  		__A   : Union[str, Any]								= FlaxOPTForCausalLM.from_pretrained(lowerCamelCase__       )
  
  		__A   : List[str]								= "left"
  
  		# use different length sentences to test batching
  		__A   : Tuple								= [
  		    "Hello, my dog is a little",
  		    "Today, I",
  		]
  
  		__A   : int								= tokenizer(lowerCamelCase__				,			return_tensors="""jax"""				,			padding=lowerCamelCase__       )
  		__A   : List[str]								= inputs["input_ids"]
  
  		__A   : Tuple								= model.generate(input_ids=lowerCamelCase__				,			attention_mask=inputs["""attention_mask"""]				,			trace=lowerCamelCase__       )
  
  		__A   : List[Any]								= tokenizer(sentences[0]				,			return_tensors="""jax"""       ).input_ids
  		__A   : Union[str, Any]								= model.generate(input_ids=lowerCamelCase__       )
  
  		__A   : Tuple								= inputs_non_padded.shape[-1] - inputs["attention_mask"][-1].sum()
  		__A   : Dict								= tokenizer(sentences[1]				,			return_tensors="""jax"""       ).input_ids
  		__A   : int								= model.generate(input_ids=lowerCamelCase__				,			max_length=model.config.max_length - num_paddings       )
  
  		__A   : List[str]								= tokenizer.batch_decode(outputs[0]				,			skip_special_tokens=lowerCamelCase__       )
  		__A   : Tuple								= tokenizer.decode(output_non_padded[0][0]				,			skip_special_tokens=lowerCamelCase__       )
  		__A   : Tuple								= tokenizer.decode(output_padded[0][0]				,			skip_special_tokens=lowerCamelCase__       )
  
  		__A   : Union[str, Any]								= [
  		    "Hello, my dog is a little bit of a dork.\nI'm a little bit",
  		    "Today, I was in the middle of a conversation with a friend about the",
  		]
  		self.assertListEqual(lowerCamelCase__				,			lowerCamelCase__       )
  		self.assertListEqual(lowerCamelCase__				,			[non_padded_sentence, padded_sentence]       )
  
  
  
  
  
  
style_context: 
  
  import os
  from typing import List, Union
  
  import tensorflow as tf
  from tensorflow_text import BertTokenizer as BertTokenizerLayer
  from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs
  
  from .tokenization_bert import BertTokenizer
  
  
  
  
  
  
  class     lowerCamelCase_			(  tf.keras.layers.Layer  ):
  
  
  
  
  	def __init__(    self	:    int				,			__A	:    List				,			__A	:    bool				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    str = "longest"				,			__A	:    bool = True				,			__A	:    int = 512				,			__A	:    int = None				,			__A	:    bool = True				,			__A	:    bool = True				,			__A	:    bool = True				,			**__A	:    Any				,			):
  		super().__init__()
  		if use_fast_bert_tokenizer:
  			__A   : Optional[int]								= FastBertTokenizer(
  			    __A				,			token_out_type=tf.intaa				,			lower_case_nfd_strip_accents=__A				,			**__A       )
  		else:
  			__A   : Optional[Any]								= tf.lookup.StaticVocabularyTable(
  			    tf.lookup.KeyValueTensorInitializer(
  			        keys=__A				,			key_dtype=tf.string				,			values=tf.range(tf.size(__A				,			out_type=tf.intaa       )				,			dtype=tf.intaa       )				,			value_dtype=tf.intaa				,			)				,			num_oov_buckets=1				,			)
  			__A   : Any								= BertTokenizerLayer(
  			    __A				,			token_out_type=tf.intaa				,			lower_case=__A				,			**__A       )
  
  		__A   : Union[str, Any]								= vocab_list
  		__A   : List[str]								= do_lower_case
  		__A   : List[Any]								= cls_token_id or vocab_list.index("""[CLS]"""       )
  		__A   : Tuple								= sep_token_id or vocab_list.index("""[SEP]"""       )
  		__A   : str								= pad_token_id or vocab_list.index("""[PAD]"""       )
  		__A   : Optional[int]								= ShrinkLongestTrimmer(max_length - 3				,			axis=1       )  # Allow room for special tokens
  		__A   : Union[str, Any]								= max_length
  		__A   : List[Any]								= padding
  		__A   : Optional[int]								= truncation
  		__A   : Tuple								= pad_to_multiple_of
  		__A   : Optional[Any]								= return_token_type_ids
  		__A   : int								= return_attention_mask
  
  
  
  
  	@classmethod
  	def 					lowerCAmelCase_					(    cls	:    List[Any]				,			__A	:    "PreTrainedTokenizerBase"				,			**__A	:    List[str]       ):  # noqa: F821
  		__A   : Any								= kwargs.pop("""do_lower_case"""				,			__A       )
  		__A   : Dict								= tokenizer.do_lower_case if do_lower_case is None else do_lower_case
  		__A   : List[str]								= kwargs.pop("""cls_token_id"""				,			__A       )
  		__A   : str								= tokenizer.cls_token_id if cls_token_id is None else cls_token_id
  		__A   : str								= kwargs.pop("""sep_token_id"""				,			__A       )
  		__A   : Dict								= tokenizer.sep_token_id if sep_token_id is None else sep_token_id
  		__A   : Tuple								= kwargs.pop("""pad_token_id"""				,			__A       )
  		__A   : List[Any]								= tokenizer.pad_token_id if pad_token_id is None else pad_token_id
  
  		__A   : Union[str, Any]								= tokenizer.get_vocab()
  		__A   : Any								= sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]				,			key=lambda __A       : x[1]       )
  		__A   : int								= [entry[0] for entry in vocab]
  		return cls(
  		    vocab_list=__A				,			do_lower_case=__A				,			cls_token_id=__A				,			sep_token_id=__A				,			pad_token_id=__A				,			**__A				,			)
  
  
  
  
  	@classmethod
  	def 					lowerCAmelCase_					(    cls	:    List[str]				,			__A	:    Union[str, os.PathLike]				,			*__A	:    Tuple				,			**__A	:    Optional[int]       ):
  		try:
  			__A   : int								= BertTokenizer.from_pretrained(__A				,			*__A				,			**__A       )
  		except:  # noqa: E722
  			from .tokenization_bert_fast import BertTokenizerFast
  
  			__A   : Optional[int]								= BertTokenizerFast.from_pretrained(__A				,			*__A				,			**__A       )
  		return cls.from_tokenizer(__A				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			__A	:    Optional[Any]       ):
  		if self.do_lower_case:
  			__A   : Tuple								= case_fold_utfa(__A       )
  		__A   : Optional[int]								= self.tf_tokenizer.tokenize(__A       )
  		return tokens.merge_dims(1				,			-1       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    Optional[Any]				,			__A	:    Dict=None				,			__A	:    List[Any]=None				,			__A	:    str=None				,			__A	:    Union[str, Any]=None				,			__A	:    Union[str, Any]=None				,			__A	:    int=None				,			__A	:    Any=None				,			):
  		if padding is None:
  			__A   : List[Any]								= self.padding
  		if padding not in ("longest", "max_length"):
  			raise ValueError("""Padding must be either 'longest' or 'max_length'!"""       )
  		if max_length is not None and text_pair is not None:
  			# Because we have to instantiate a Trimmer to do it properly
  			raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""       )
  		if max_length is None:
  			__A   : Any								= self.max_length
  		if truncation is None:
  			__A   : int								= self.truncation
  		if pad_to_multiple_of is None:
  			__A   : List[str]								= self.pad_to_multiple_of
  		if return_token_type_ids is None:
  			__A   : int								= self.return_token_type_ids
  		if return_attention_mask is None:
  			__A   : Any								= self.return_attention_mask
  		if not isinstance(__A				,			tf.Tensor       ):
  			__A   : str								= tf.convert_to_tensor(__A       )
  		if text_pair is not None and not isinstance(__A				,			tf.Tensor       ):
  			__A   : Optional[Any]								= tf.convert_to_tensor(__A       )
  		if text_pair is not None:
  			if text.shape.rank > 1:
  				raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""       )
  			if text_pair.shape.rank > 1:
  				raise ValueError("""text_pair should not be multidimensional!"""       )
  		if text.shape.rank == 2:
  			__A						,							__A   : Optional[int]								= text[:, 0], text[:, 1]
  		__A   : List[Any]								= self.unpaired_tokenize(__A       )
  		if text_pair is None:  # Unpaired text
  			if truncation:
  				__A   : Union[str, Any]								= text[:, : max_length - 2]  # Allow room for special tokens
  			__A						,							__A   : List[Any]								= combine_segments(
  			    (text,)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
  		else:  # Paired text
  			__A   : Tuple								= self.unpaired_tokenize(__A       )
  			if truncation:
  				__A						,							__A   : Optional[int]								= self.paired_trimmer.trim([text, text_pair]       )
  			__A						,							__A   : int								= combine_segments(
  			    (text, text_pair)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
  		if padding == "longest":
  			__A   : str								= input_ids.bounding_shape(axis=1       )
  			if pad_to_multiple_of is not None:
  				# No ceiling division in tensorflow, so we negate floordiv instead
  				__A   : Optional[Any]								= pad_to_multiple_of * (-tf.math.floordiv(-pad_length				,			__A       ))
  		else:
  			__A   : Dict								= max_length
  
  		__A						,							__A   : str								= pad_model_inputs(__A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
  		__A   : List[str]								= {"""input_ids""": input_ids}
  		if return_attention_mask:
  			__A   : Any								= attention_mask
  		if return_token_type_ids:
  			__A						,							__A   : Tuple								= pad_model_inputs(
  			    __A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
  			__A   : List[str]								= token_type_ids
  		return output
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
  		return {
  		    "vocab_list": self.vocab_list,
  		    "do_lower_case": self.do_lower_case,
  		    "cls_token_id": self.cls_token_id,
  		    "sep_token_id": self.sep_token_id,
  		    "pad_token_id": self.pad_token_id,
  		}
  
  
  
  
  
  