






import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase  							=					get_tests_dir("fixtures/spiece.model")




@require_sentencepiece
@require_tokenizers
class  UpperCamelCase_      (			snake_case_       ,					unittest.TestCase				):



	'''simple docstring'''




	lowerCAmelCase													= AlbertTokenizer
	lowerCAmelCase													= AlbertTokenizerFast
	lowerCAmelCase													= True
	lowerCAmelCase													= True
	lowerCAmelCase													= True




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								snake_case_				=							AlbertTokenizer(a					)
								tokenizer.save_pretrained(self.tmpdirname					)




	def 							_UpperCamelCase			(       self       ,			a					)       ->    str:
								snake_case_				=							'this is a test'
								snake_case_				=							'this is a test'
								return input_text, output_text




	def 							_UpperCamelCase			(       self					)       ->    int:
								snake_case_				=							'<pad>'
								snake_case_				=							0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(a					)       ,			a					)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(a					)       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								snake_case_				=							list(self.get_tokenizer().get_vocab().keys()					)

								self.assertEqual(vocab_keys[0]       ,			'<pad>'					)
								self.assertEqual(vocab_keys[1]       ,			'<unk>'					)
								self.assertEqual(vocab_keys[-1]       ,			'▁eloquent'					)
								self.assertEqual(len(a					)       ,			3_00_00					)




	def 							_UpperCamelCase			(       self					)       ->    Any:
								self.assertEqual(self.get_tokenizer().vocab_size       ,			3_00_00					)




	def 							_UpperCamelCase			(       self					)       ->    Tuple:
								if not self.test_rust_tokenizer:
															return

								snake_case_				=							self.get_tokenizer()
								snake_case_				=							self.get_rust_tokenizer()

								snake_case_				=							'I was born in 92000, and this is falsé.'

								snake_case_				=							tokenizer.tokenize(a					)
								snake_case_				=							rust_tokenizer.tokenize(a					)
								self.assertListEqual(a       ,			a					)

								snake_case_				=							tokenizer.encode(a       ,			add_special_tokens=a					)
								snake_case_				=							rust_tokenizer.encode(a       ,			add_special_tokens=a					)
								self.assertListEqual(a       ,			a					)

								snake_case_				=							self.get_rust_tokenizer()
								snake_case_				=							tokenizer.encode(a					)
								snake_case_				=							rust_tokenizer.encode(a					)
								self.assertListEqual(a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    List[str]:
								snake_case_				=							AlbertTokenizer(a       ,			keep_accents=a					)

								snake_case_				=							tokenizer.tokenize('This is a test'					)
								self.assertListEqual(a       ,			['▁this', '▁is', '▁a', '▁test']					)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(a					)       ,			[48, 25, 21, 12_89]					)

								snake_case_				=							tokenizer.tokenize('I was born in 92000, and this is falsé.'					)
								self.assertListEqual(
								    a       ,			['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']					)
								snake_case_				=							tokenizer.convert_tokens_to_ids(a					)
								self.assertListEqual(a       ,			[31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]					)

								snake_case_				=							tokenizer.convert_ids_to_tokens(a					)
								self.assertListEqual(
								    a       ,			['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']       ,			)




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_				=							AlbertTokenizer(a					)

								snake_case_				=							tokenizer.encode('sequence builders'					)
								snake_case_				=							tokenizer.encode('multi-sequence build'					)

								snake_case_				=							tokenizer.build_inputs_with_special_tokens(a					)
								snake_case_				=							tokenizer.build_inputs_with_special_tokens(a       ,			a					)

								assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
								assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
								    tokenizer.sep_token_id
								]






	@slow
	def 							_UpperCamelCase			(       self					)       ->    int:
								# fmt: off
								snake_case_				=							{'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=a       ,			model_name='albert-base-v2'       ,			revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'       ,			)













import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
	import torch

	from transformers import AutoModelForImageClassification

if is_vision_available():
	from transformers import AutoImageProcessor




@require_torch
@require_vision
class  UpperCamelCase_      (			unittest.TestCase				):



	'''simple docstring'''




	@slow
	def 							_UpperCamelCase			(       self					)       ->    int:
								snake_case_				=							AutoImageProcessor.from_pretrained('microsoft/dit-base-finetuned-rvlcdip'					)
								snake_case_				=							AutoModelForImageClassification.from_pretrained('microsoft/dit-base-finetuned-rvlcdip'					)
								model.to(a					)

								from datasets import load_dataset

								snake_case_				=							load_dataset('nielsr/rvlcdip-demo'					)

								snake_case_				=							dataset['train'][0]['image'].convert('RGB'					)

								snake_case_				=							image_processor(a       ,			return_tensors='pt'					).to(a					)

								# forward pass
								with torch.no_grad():
															snake_case_				=							model(**a					)
															snake_case_				=							outputs.logits

								snake_case_				=							torch.Size((1, 16)					)
								self.assertEqual(logits.shape       ,			a					)

								snake_case_				=							torch.tensor(
								    [-0.4_158, -0.4_092, -0.4_347]       ,			device=a       ,			dtype=torch.float       ,			)
								self.assertTrue(torch.allclose(logits[0, :3]       ,			a       ,			atol=1E-4					)					)





