
from __future__ import annotations

import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import is_tf_available, is_vision_available

from ...test_modeling_tf_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_tf_bert import TFBertModelTester
from ..clip.test_modeling_tf_clip import TFCLIPVisionModelTester
from ..deit.test_modeling_tf_deit import TFDeiTModelTester
from ..roberta.test_modeling_tf_roberta import TFRobertaModelTester
from ..vit.test_modeling_tf_vit import TFViTModelTester


if is_tf_available():
					from transformers import (
					    TFBertModel,
					    TFCLIPVisionModel,
					    TFDeiTModel,
					    TFRobertaModel,
					    TFVisionTextDualEncoderModel,
					    TFViTModel,
					    VisionTextDualEncoderConfig,
					)

if is_vision_available():
					from PIL import Image

					from transformers import VisionTextDualEncoderProcessor
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Tuple:
			if isinstance(SCREAMING_SNAKE_CASE_       ,							collections.abc.Iterable     ):
						return x
			return (x, x)







@require_tf
class      A__							:






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      Dict    ,     a					:      Union[str, Any]   ):






							'''simple docstring'''







							pass






				def 							_lowerCamelCase		(						self					:      Dict   ):






							'''simple docstring'''







							pass






				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							pass






				def 							_lowerCamelCase		(						self					:      Optional[Any]    ,     a					:      Dict    ,     a					:      List[Any]    ,     a					:      List[str]    ,     a					:      Dict    ,     a					:      int=None    ,     **a					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		VisionTextDualEncoderConfig.from_vision_text_configs(a    ,     a   )

							lowerCAmelCase__      :      Optional[Any]         =		TFVisionTextDualEncoderModel(a   )

							lowerCAmelCase__      :      Tuple         =		model(input_ids=a    ,     pixel_values=a    ,     attention_mask=a   )

							self.assertEqual(output['text_embeds'].shape    ,     (input_ids.shape[0], config.projection_dim)   )
							self.assertEqual(output['image_embeds'].shape    ,     (pixel_values.shape[0], config.projection_dim)   )






				def 							_lowerCamelCase		(						self					:      Tuple    ,     a					:      List[str]    ,     a					:      Tuple    ,     a					:      List[Any]    ,     a					:      List[Any]    ,     a					:      Union[str, Any]=None    ,     **a					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[int]         =		self.get_vision_text_model(a    ,     a   )
							lowerCAmelCase__      :      Optional[int]         =		TFVisionTextDualEncoderModel(vision_model=a    ,     text_model=a   )

							lowerCAmelCase__      :      Optional[Any]         =		model(input_ids=a    ,     pixel_values=a    ,     attention_mask=a   )

							self.assertEqual(output['text_embeds'].shape    ,     (input_ids.shape[0], model.config.projection_dim)   )
							self.assertEqual(output['image_embeds'].shape    ,     (pixel_values.shape[0], model.config.projection_dim)   )






				def 							_lowerCamelCase		(						self					:      Dict    ,     a					:      Tuple    ,     a					:      Dict    ,     a					:      Optional[int]    ,     a					:      Optional[int]    ,     a					:      Dict=None    ,     **a					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      Tuple         =		self.get_vision_text_model(a    ,     a   )
							lowerCAmelCase__      :      str         =		{'vision_model': vision_model, 'text_model': text_model}
							lowerCAmelCase__      :      Tuple         =		TFVisionTextDualEncoderModel.from_vision_text_pretrained(**a   )

							lowerCAmelCase__      :      int         =		model(input_ids=a    ,     pixel_values=a    ,     attention_mask=a   )

							self.assertEqual(output['text_embeds'].shape    ,     (input_ids.shape[0], model.config.projection_dim)   )
							self.assertEqual(output['image_embeds'].shape    ,     (pixel_values.shape[0], model.config.projection_dim)   )






				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      List[str]    ,     a					:      Optional[Any]    ,     a					:      Union[str, Any]    ,     a					:      Optional[int]    ,     a					:      str=None    ,     **a					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[int]         =		self.get_vision_text_model(a    ,     a   )
							lowerCAmelCase__      :      int         =		TFVisionTextDualEncoderModel(vision_model=a    ,     text_model=a   )

							lowerCAmelCase__      :      Dict         =		model(input_ids=a    ,     pixel_values=a    ,     attention_mask=a   )
							lowerCAmelCase__      :      int         =		output[0].numpy()

							with tempfile.TemporaryDirectory() as tmpdirname:
										model.save_pretrained(a   )
										lowerCAmelCase__      :      List[str]         =		TFVisionTextDualEncoderModel.from_pretrained(a   )

										lowerCAmelCase__      :      List[Any]         =		model(input_ids=a    ,     pixel_values=a    ,     attention_mask=a   )
										lowerCAmelCase__      :      List[str]         =		after_output[0].numpy()
										lowerCAmelCase__      :      Optional[int]         =		np.amax(np.abs(out_a - out_a   )   )
										self.assertLessEqual(a    ,     1E-5   )






				def 							_lowerCamelCase		(						self					:      Optional[int]    ,     a					:      Tuple    ,     a					:      List[str]    ,     a					:      str    ,     a					:      Any    ,     a					:      Any=None    ,     **a					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      Dict         =		self.get_vision_text_model(a    ,     a   )
							lowerCAmelCase__      :      Any         =		TFVisionTextDualEncoderModel(vision_model=a    ,     text_model=a   )

							lowerCAmelCase__      :      List[Any]         =		model(
							    input_ids=a    ,     pixel_values=a    ,     attention_mask=a    ,     output_attentions=a   )

							lowerCAmelCase__      :      Any         =		output.vision_model_output.attentions
							self.assertEqual(len(a   )    ,     vision_config.num_hidden_layers   )

							# in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
							lowerCAmelCase__      :      int         =		to_atuple(vision_model.config.image_size   )
							lowerCAmelCase__      :      Union[str, Any]         =		to_atuple(vision_model.config.patch_size   )
							lowerCAmelCase__      :      Optional[Any]         =		(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
							lowerCAmelCase__      :      List[Any]         =		num_patches + 1
							self.assertEqual(vision_attentions[0].shape[-3:]    ,     (vision_config.num_attention_heads, seq_len, seq_len)   )

							lowerCAmelCase__      :      str         =		output.text_model_output.attentions
							self.assertEqual(len(a   )    ,     text_config.num_hidden_layers   )

							self.assertEqual(
							    text_attentions[0].shape[-3:]    ,     (text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])    ,     )






				def 							_lowerCamelCase		(						self					:      List[Any]    ,     a					:      np.ndarray    ,     a					:      np.ndarray    ,     a					:      float   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		np.abs((a - b)   ).max()
							self.assertLessEqual(a    ,     a    ,     f'''Difference between torch and flax is {diff} (>= {tol}).'''   )






				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		self.prepare_config_and_inputs()
							self.check_vision_text_dual_encoder_model(**a   )






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		self.prepare_config_and_inputs()
							self.check_model_from_pretrained_configs(**a   )






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      str         =		self.prepare_config_and_inputs()
							self.check_vision_text_dual_encoder_from_pretrained(**a   )






				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		self.prepare_config_and_inputs()
							self.check_save_load(**a   )






				def 							_lowerCamelCase		(						self					:      Dict   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		self.prepare_config_and_inputs()
							self.check_vision_text_output_attention(**a   )






				@slow
				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      Dict         =		self.get_pretrained_model_and_inputs()

							lowerCAmelCase__      :      str         =		model_a(**a   )
							lowerCAmelCase__      :      Optional[int]         =		outputs[0].numpy()

							with tempfile.TemporaryDirectory() as tmp_dirname:
										model_a.save_pretrained(a   )
										lowerCAmelCase__      :      Any         =		TFVisionTextDualEncoderModel.from_pretrained(a   )

										lowerCAmelCase__      :      Optional[int]         =		model_a(**a   )
										lowerCAmelCase__      :      List[str]         =		after_outputs[0].numpy()
										lowerCAmelCase__      :      List[str]         =		np.amax(np.abs(out_a - out_a   )   )
										self.assertLessEqual(a    ,     1E-5   )







@require_tf
class      A__							(     __magic_name__						,   unittest.TestCase	):






				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      str         =		TFVisionTextDualEncoderModel.from_vision_text_pretrained(
							    'hf-internal-testing/tiny-random-vit'    ,     'hf-internal-testing/tiny-random-bert'   )
							lowerCAmelCase__      :      str         =		13
							lowerCAmelCase__      :      Dict         =		floats_tensor(
							    [
							        batch_size,
							        model.vision_model.config.num_channels,
							        model.vision_model.config.image_size,
							        model.vision_model.config.image_size,
							    ]   )
							lowerCAmelCase__      :      Optional[int]         =		ids_tensor([batch_size, 4]    ,     model.text_model.config.vocab_size   )
							lowerCAmelCase__      :      Any         =		random_attention_mask([batch_size, 4]   )
							lowerCAmelCase__      :      Optional[Any]         =		{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

							return model, inputs






				def 							_lowerCamelCase		(						self					:      str    ,     a					:      Optional[Any]    ,     a					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      str         =		TFViTModel(a    ,     name='vision_model'   )
							lowerCAmelCase__      :      Optional[int]         =		TFBertModel(a    ,     name='text_model'   )
							return vision_model, text_model






				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		TFViTModelTester(self   )
							lowerCAmelCase__      :      str         =		TFBertModelTester(self   )
							lowerCAmelCase__      :      List[str]         =		vit_model_tester.prepare_config_and_inputs()
							lowerCAmelCase__      :      Optional[Any]         =		bert_model_tester.prepare_config_and_inputs()

							lowerCAmelCase__      ,				lowerCAmelCase__      ,				lowerCAmelCase__      :      str         =		vision_config_and_inputs

							(
							    (
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				
							)      :      Tuple         =		text_config_and_inputs

							return {
							    "text_config": text_config,
							    "vision_config": vision_config,
							    "pixel_values": pixel_values,
							    "attention_mask": input_mask,
							    "input_ids": input_ids,
							    "text_token_type_ids": token_type_ids,
							    "text_sequence_labels": sequence_labels,
							    "text_token_labels": token_labels,
							    "text_choice_labels": choice_labels,
							}







@require_tf
class      A__							(     __magic_name__						,   unittest.TestCase	):






				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		TFVisionTextDualEncoderModel.from_vision_text_pretrained(
							    'Rocketknight1/tiny-random-deit-tf'    ,     'hf-internal-testing/tiny-random-roberta'   )
							lowerCAmelCase__      :      List[str]         =		13
							lowerCAmelCase__      :      Dict         =		floats_tensor(
							    [
							        batch_size,
							        model.vision_model.config.num_channels,
							        model.vision_model.config.image_size,
							        model.vision_model.config.image_size,
							    ]   )
							lowerCAmelCase__      :      List[Any]         =		ids_tensor([batch_size, 4]    ,     model.text_model.config.vocab_size   )
							lowerCAmelCase__      :      Union[str, Any]         =		random_attention_mask([batch_size, 4]   )
							lowerCAmelCase__      :      str         =		{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

							return model, inputs






				def 							_lowerCamelCase		(						self					:      str    ,     a					:      Optional[int]    ,     a					:      Union[str, Any]    ,     a					:      Dict    ,     a					:      str    ,     a					:      Union[str, Any]=None    ,     **a					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      List[str]         =		self.get_vision_text_model(a    ,     a   )
							lowerCAmelCase__      :      Union[str, Any]         =		TFVisionTextDualEncoderModel(vision_model=a    ,     text_model=a   )

							lowerCAmelCase__      :      Any         =		model(
							    input_ids=a    ,     pixel_values=a    ,     attention_mask=a    ,     output_attentions=a   )

							lowerCAmelCase__      :      str         =		output.vision_model_output.attentions
							self.assertEqual(len(a   )    ,     vision_config.num_hidden_layers   )

							# in DEiT, the seq_len equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
							lowerCAmelCase__      :      Union[str, Any]         =		to_atuple(vision_model.config.image_size   )
							lowerCAmelCase__      :      int         =		to_atuple(vision_model.config.patch_size   )
							lowerCAmelCase__      :      Dict         =		(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
							lowerCAmelCase__      :      Optional[Any]         =		num_patches + 2
							self.assertEqual(vision_attentions[0].shape[-3:]    ,     (vision_config.num_attention_heads, seq_len, seq_len)   )

							lowerCAmelCase__      :      Optional[int]         =		output.text_model_output.attentions
							self.assertEqual(len(a   )    ,     text_config.num_hidden_layers   )

							self.assertEqual(
							    text_attentions[0].shape[-3:]    ,     (text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])    ,     )






				def 							_lowerCamelCase		(						self					:      Optional[Any]    ,     a					:      List[Any]    ,     a					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		TFDeiTModel(a    ,     name='vision_model'   )
							lowerCAmelCase__      :      Optional[Any]         =		TFRobertaModel(a    ,     name='text_model'   )
							return vision_model, text_model






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		TFDeiTModelTester(self   )
							lowerCAmelCase__      :      int         =		TFRobertaModelTester(self   )
							lowerCAmelCase__      :      Optional[int]         =		vit_model_tester.prepare_config_and_inputs()
							lowerCAmelCase__      :      Optional[Any]         =		bert_model_tester.prepare_config_and_inputs()

							lowerCAmelCase__      ,				lowerCAmelCase__      ,				lowerCAmelCase__      :      Any         =		vision_config_and_inputs

							(
							    (
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				
							)      :      Tuple         =		text_config_and_inputs

							return {
							    "text_config": text_config,
							    "vision_config": vision_config,
							    "pixel_values": pixel_values,
							    "attention_mask": input_mask,
							    "input_ids": input_ids,
							    "text_token_type_ids": token_type_ids,
							    "text_sequence_labels": sequence_labels,
							    "text_token_labels": token_labels,
							    "text_choice_labels": choice_labels,
							}







@require_tf
class      A__							(     __magic_name__						,   unittest.TestCase	):






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		TFVisionTextDualEncoderModel.from_vision_text_pretrained(
							    'Rocketknight1/tiny-random-clip-tf'    ,     'hf-internal-testing/tiny-random-bert'   )
							lowerCAmelCase__      :      Union[str, Any]         =		13
							lowerCAmelCase__      :      str         =		floats_tensor(
							    [
							        batch_size,
							        model.vision_model.config.num_channels,
							        model.vision_model.config.image_size,
							        model.vision_model.config.image_size,
							    ]   )
							lowerCAmelCase__      :      List[str]         =		ids_tensor([batch_size, 4]    ,     model.text_model.config.vocab_size   )
							lowerCAmelCase__      :      str         =		random_attention_mask([batch_size, 4]   )
							lowerCAmelCase__      :      int         =		{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

							return model, inputs






				def 							_lowerCamelCase		(						self					:      Tuple    ,     a					:      List[str]    ,     a					:      Dict   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		TFCLIPVisionModel(a    ,     name='vision_model'   )
							lowerCAmelCase__      :      int         =		TFBertModel(a    ,     name='text_model'   )
							return vision_model, text_model






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[int]         =		TFCLIPVisionModelTester(self   )
							lowerCAmelCase__      :      Any         =		TFBertModelTester(self   )
							lowerCAmelCase__      :      Optional[int]         =		clip_model_tester.prepare_config_and_inputs()
							lowerCAmelCase__      :      int         =		bert_model_tester.prepare_config_and_inputs()

							lowerCAmelCase__      ,				lowerCAmelCase__      :      Any         =		vision_config_and_inputs

							(
							    (
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				
							)      :      str         =		text_config_and_inputs

							return {
							    "text_config": text_config,
							    "vision_config": vision_config,
							    "pixel_values": pixel_values,
							    "attention_mask": input_mask,
							    "input_ids": input_ids,
							    "text_token_type_ids": token_type_ids,
							    "text_sequence_labels": sequence_labels,
							    "text_token_labels": token_labels,
							    "text_choice_labels": choice_labels,
							}







@require_vision
@require_tf
class      A__							(     unittest.TestCase	):






				@slow
				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[int]         =		TFVisionTextDualEncoderModel.from_pretrained(
							    'clip-italian/clip-italian'    ,     logit_scale_init_value=1.0    ,     from_pt=a   )
							lowerCAmelCase__      :      Optional[int]         =		VisionTextDualEncoderProcessor.from_pretrained('clip-italian/clip-italian'   )

							lowerCAmelCase__      :      Union[str, Any]         =		Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'   )
							lowerCAmelCase__      :      Dict         =		processor(
							    text=['una foto di un gatto', 'una foto di un cane']    ,     images=a    ,     padding=a    ,     return_tensors='np'   )

							lowerCAmelCase__      :      List[str]         =		model(**a   )

							# verify the logits
							self.assertEqual(outputs.logits_per_image.shape    ,     (inputs.pixel_values.shape[0], inputs.input_ids.shape[0])   )
							self.assertEqual(
							    outputs.logits_per_text.shape    ,     (inputs.input_ids.shape[0], inputs.pixel_values.shape[0])    ,     )

							lowerCAmelCase__      :      Tuple         =		np.array([[1.2_2_8_4_7_2_7, 0.3_1_0_4_1_2_2]]   )

							self.assertTrue(np.allclose(outputs.logits_per_image.numpy()    ,     a    ,     atol=1E-3   )   )

import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__          =		get_tests_dir("""fixtures/spiece.model""")







@require_sentencepiece
@require_tokenizers
class      A__							(     __magic_name__						,   unittest.TestCase	):
				lowercase			    =  AlbertTokenizer
				lowercase			    =  AlbertTokenizerFast
				lowercase			    =  True
				lowercase			    =  True
				lowercase			    =  True






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCAmelCase__      :      int         =		AlbertTokenizer(a   )
							tokenizer.save_pretrained(self.tmpdirname   )






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		'this is a test'
							lowerCAmelCase__      :      List[Any]         =		'this is a test'
							return input_text, output_text






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		'<pad>'
							lowerCAmelCase__      :      Optional[Any]         =		0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(a   )    ,     a   )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(a   )    ,     a   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		list(self.get_tokenizer().get_vocab().keys()   )

							self.assertEqual(vocab_keys[0]    ,     '<pad>'   )
							self.assertEqual(vocab_keys[1]    ,     '<unk>'   )
							self.assertEqual(vocab_keys[-1]    ,     '▁eloquent'   )
							self.assertEqual(len(a   )    ,     30_000   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							self.assertEqual(self.get_tokenizer().vocab_size    ,     30_000   )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							if not self.test_rust_tokenizer:
										return

							lowerCAmelCase__      :      str         =		self.get_tokenizer()
							lowerCAmelCase__      :      str         =		self.get_rust_tokenizer()

							lowerCAmelCase__      :      List[Any]         =		'I was born in 92000, and this is falsé.'

							lowerCAmelCase__      :      str         =		tokenizer.tokenize(a   )
							lowerCAmelCase__      :      Optional[int]         =		rust_tokenizer.tokenize(a   )
							self.assertListEqual(a    ,     a   )

							lowerCAmelCase__      :      Tuple         =		tokenizer.encode(a    ,     add_special_tokens=a   )
							lowerCAmelCase__      :      Union[str, Any]         =		rust_tokenizer.encode(a    ,     add_special_tokens=a   )
							self.assertListEqual(a    ,     a   )

							lowerCAmelCase__      :      Optional[Any]         =		self.get_rust_tokenizer()
							lowerCAmelCase__      :      Dict         =		tokenizer.encode(a   )
							lowerCAmelCase__      :      List[Any]         =		rust_tokenizer.encode(a   )
							self.assertListEqual(a    ,     a   )






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		AlbertTokenizer(a    ,     keep_accents=a   )

							lowerCAmelCase__      :      Union[str, Any]         =		tokenizer.tokenize('This is a test'   )
							self.assertListEqual(a    ,     ['▁this', '▁is', '▁a', '▁test']   )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(a   )    ,     [48, 25, 21, 1_289]   )

							lowerCAmelCase__      :      Tuple         =		tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
							self.assertListEqual(
							    a    ,     ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']   )
							lowerCAmelCase__      :      Any         =		tokenizer.convert_tokens_to_ids(a   )
							self.assertListEqual(a    ,     [31, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]   )

							lowerCAmelCase__      :      Any         =		tokenizer.convert_ids_to_tokens(a   )
							self.assertListEqual(
							    a    ,     ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']    ,     )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		AlbertTokenizer(a   )

							lowerCAmelCase__      :      Tuple         =		tokenizer.encode('sequence builders'   )
							lowerCAmelCase__      :      Any         =		tokenizer.encode('multi-sequence build'   )

							lowerCAmelCase__      :      Dict         =		tokenizer.build_inputs_with_special_tokens(a   )
							lowerCAmelCase__      :      Tuple         =		tokenizer.build_inputs_with_special_tokens(a    ,     a   )

							assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
							assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
							    tokenizer.sep_token_id
							]






				@slow
				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		{'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 21_970, 13, 5, 6_092, 167, 28, 7_103, 2_153, 673, 8, 7_028, 12_051, 18, 17, 7_103, 2_153, 673, 8, 3_515, 18_684, 8, 4_461, 6, 1_927, 297, 8, 12_060, 2_607, 18, 13, 5, 4_461, 15, 10_538, 38, 8, 135, 15, 822, 58, 15, 993, 10_363, 15, 1_460, 8_005, 4_461, 15, 993, 255, 2_328, 9, 9, 9, 6, 26, 1_112, 816, 3_260, 13, 5, 103, 2_377, 6, 17, 1_112, 816, 2_782, 13, 5, 103, 10_641, 6, 29, 84, 2_512, 2_430, 782, 18_684, 2_761, 19, 808, 2_430, 2_556, 17, 855, 1_480, 9_477, 4_091, 128, 11_712, 15, 7_103, 2_153, 673, 17, 24_883, 9_990, 9, 3], [2, 11_502, 25, 1_006, 20, 782, 8, 11_809, 855, 1_732, 19_393, 18_667, 37, 367, 21_018, 69, 1_854, 34, 11_860, 19_124, 27, 156, 225, 17, 193, 4_141, 19, 65, 9_124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2_231, 886, 2_385, 17_659, 84, 14, 16_792, 1_952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=a    ,     model_name='albert-base-v2'    ,     revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'    ,     )