import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					lowercase__						(   UpperCamelCase_	,							unittest.TestCase):
						UpperCamelCase_     =		LxmertTokenizer
						UpperCamelCase_     =		LxmertTokenizerFast
						UpperCamelCase_     =		True
						UpperCamelCase_     =		True


						def        __A       (       self	:				List[str]						):





												'''simple docstring'''





												super().setUp()

												SCREAMING_SNAKE_CASE							:    Any							=							[
												    '''[UNK]''',
												    '''[CLS]''',
												    '''[SEP]''',
												    '''want''',
												    '''##want''',
												    '''##ed''',
												    '''wa''',
												    '''un''',
												    '''runn''',
												    '''##ing''',
												    ''',''',
												    '''low''',
												    '''lowest''',
												]
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['''vocab_file''']						)
												with open(self.vocab_file    ,		'''w'''    ,		encoding='''utf-8'''						) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)


						def        __A       (       self	:				str    ,		UpperCamelCase__	:				Dict						):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    List[str]							=							'''UNwant\u00E9d,running'''
												SCREAMING_SNAKE_CASE							:    List[str]							=							'''unwanted, running'''
												return input_text, output_text


						def        __A       (       self	:				Optional[Any]						):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    Dict							=							self.tokenizer_class(self.vocab_file						)

												SCREAMING_SNAKE_CASE							:    List[str]							=							tokenizer.tokenize('''UNwant\u00E9d,running'''						)
												self.assertListEqual(UpperCamelCase__    ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']						)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)    ,		[7, 4, 5, 10, 8, 9]						)



						def        __A       (       self	:				Optional[int]						):





												'''simple docstring'''





												if not self.test_rust_tokenizer:
																		return

												SCREAMING_SNAKE_CASE							:    Tuple							=							self.get_tokenizer()
												SCREAMING_SNAKE_CASE							:    Any							=							self.get_rust_tokenizer()

												SCREAMING_SNAKE_CASE							:    str							=							'''I was born in 92000, and this is falsé.'''

												SCREAMING_SNAKE_CASE							:    List[str]							=							tokenizer.tokenize(UpperCamelCase__						)
												SCREAMING_SNAKE_CASE							:    str							=							rust_tokenizer.tokenize(UpperCamelCase__						)
												self.assertListEqual(UpperCamelCase__    ,		UpperCamelCase__						)

												SCREAMING_SNAKE_CASE							:    Optional[int]							=							tokenizer.encode(UpperCamelCase__    ,		add_special_tokens=UpperCamelCase__						)
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							rust_tokenizer.encode(UpperCamelCase__    ,		add_special_tokens=UpperCamelCase__						)
												self.assertListEqual(UpperCamelCase__    ,		UpperCamelCase__						)

												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							self.get_rust_tokenizer()
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							tokenizer.encode(UpperCamelCase__						)
												SCREAMING_SNAKE_CASE							:    List[str]							=							rust_tokenizer.encode(UpperCamelCase__						)
												self.assertListEqual(UpperCamelCase__    ,		UpperCamelCase__						)






import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402





class 					lowercase__						(   unittest.TestCase):


						def        __A       (       self	:				List[Any]						):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    Dict							=							0


						def        __A       (       self	:				Dict						):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    str							=							AutoImageProcessor.from_pretrained('''openai/clip-vit-base-patch32'''						)
												self.assertIsInstance(UpperCamelCase__    ,		UpperCamelCase__						)


						def        __A       (       self	:				Dict						):





												'''simple docstring'''





												with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE							:    str							=							Path(UpperCamelCase__						) / '''preprocessor_config.json'''
																		SCREAMING_SNAKE_CASE							:    Dict							=							Path(UpperCamelCase__						) / '''config.json'''
																		json.dump(
																		    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}    ,		open(UpperCamelCase__    ,		'''w'''						)    ,		)
																		json.dump({'''model_type''': '''clip'''}    ,		open(UpperCamelCase__    ,		'''w'''						)						)

																		SCREAMING_SNAKE_CASE							:    Any							=							AutoImageProcessor.from_pretrained(UpperCamelCase__						)
																		self.assertIsInstance(UpperCamelCase__    ,		UpperCamelCase__						)


						def        __A       (       self	:				int						):





												'''simple docstring'''





												with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE							:    Tuple							=							Path(UpperCamelCase__						) / '''preprocessor_config.json'''
																		SCREAMING_SNAKE_CASE							:    Dict							=							Path(UpperCamelCase__						) / '''config.json'''
																		json.dump(
																		    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}    ,		open(UpperCamelCase__    ,		'''w'''						)    ,		)
																		json.dump({'''model_type''': '''clip'''}    ,		open(UpperCamelCase__    ,		'''w'''						)						)

																		SCREAMING_SNAKE_CASE							:    Optional[Any]							=							AutoImageProcessor.from_pretrained(UpperCamelCase__						)
																		self.assertIsInstance(UpperCamelCase__    ,		UpperCamelCase__						)


						def        __A       (       self	:				List[str]						):





												'''simple docstring'''





												with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE							:    Dict							=							CLIPConfig()

																		# Create a dummy config file with image_proceesor_type
																		SCREAMING_SNAKE_CASE							:    Dict							=							Path(UpperCamelCase__						) / '''preprocessor_config.json'''
																		SCREAMING_SNAKE_CASE							:    List[str]							=							Path(UpperCamelCase__						) / '''config.json'''
																		json.dump(
																		    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}    ,		open(UpperCamelCase__    ,		'''w'''						)    ,		)
																		json.dump({'''model_type''': '''clip'''}    ,		open(UpperCamelCase__    ,		'''w'''						)						)

																		# remove image_processor_type to make sure config.json alone is enough to load image processor locally
																		SCREAMING_SNAKE_CASE							:    List[Any]							=							AutoImageProcessor.from_pretrained(UpperCamelCase__						).to_dict()

																		config_dict.pop('''image_processor_type'''						)
																		SCREAMING_SNAKE_CASE							:    str							=							CLIPImageProcessor(**UpperCamelCase__						)

																		# save in new folder
																		model_config.save_pretrained(UpperCamelCase__						)
																		config.save_pretrained(UpperCamelCase__						)

																		SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							AutoImageProcessor.from_pretrained(UpperCamelCase__						)

																		# make sure private variable is not incorrectly saved
																		SCREAMING_SNAKE_CASE							:    List[Any]							=							json.loads(config.to_json_string()						)
																		self.assertTrue('''_processor_class''' not in dict_as_saved						)

												self.assertIsInstance(UpperCamelCase__    ,		UpperCamelCase__						)


						def        __A       (       self	:				Tuple						):





												'''simple docstring'''





												with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE							:    Optional[int]							=							Path(UpperCamelCase__						) / '''preprocessor_config.json'''
																		json.dump(
																		    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}    ,		open(UpperCamelCase__    ,		'''w'''						)    ,		)

																		SCREAMING_SNAKE_CASE							:    Dict							=							AutoImageProcessor.from_pretrained(UpperCamelCase__						)
																		self.assertIsInstance(UpperCamelCase__    ,		UpperCamelCase__						)


						def        __A       (       self	:				int						):





												'''simple docstring'''





												with self.assertRaisesRegex(
												    UpperCamelCase__    ,		'''clip-base is not a local folder and is not a valid model identifier'''						):
																		SCREAMING_SNAKE_CASE							:    List[str]							=							AutoImageProcessor.from_pretrained('''clip-base'''						)


						def        __A       (       self	:				List[str]						):





												'''simple docstring'''





												with self.assertRaisesRegex(
												    UpperCamelCase__    ,		r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''						):
																		SCREAMING_SNAKE_CASE							:    Tuple							=							AutoImageProcessor.from_pretrained(UpperCamelCase__    ,		revision='''aaaaaa'''						)


						def        __A       (       self	:				Dict						):





												'''simple docstring'''





												with self.assertRaisesRegex(
												    UpperCamelCase__    ,		'''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''    ,		):
																		SCREAMING_SNAKE_CASE							:    Dict							=							AutoImageProcessor.from_pretrained('''hf-internal-testing/config-no-model'''						)


						def        __A       (       self	:				List[Any]						):





												'''simple docstring'''





												with self.assertRaises(UpperCamelCase__						):
																		SCREAMING_SNAKE_CASE							:    str							=							AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''						)
												# If remote code is disabled, we can't load this config.
												with self.assertRaises(UpperCamelCase__						):
																		SCREAMING_SNAKE_CASE							:    List[Any]							=							AutoImageProcessor.from_pretrained(
																		    '''hf-internal-testing/test_dynamic_image_processor'''    ,		trust_remote_code=UpperCamelCase__						)

												SCREAMING_SNAKE_CASE							:    List[Any]							=							AutoImageProcessor.from_pretrained(
												    '''hf-internal-testing/test_dynamic_image_processor'''    ,		trust_remote_code=UpperCamelCase__						)
												self.assertEqual(image_processor.__class__.__name__    ,		'''NewImageProcessor'''						)

												# Test image processor can be reloaded.
												with tempfile.TemporaryDirectory() as tmp_dir:
																		image_processor.save_pretrained(UpperCamelCase__						)
																		SCREAMING_SNAKE_CASE							:    Optional[int]							=							AutoImageProcessor.from_pretrained(UpperCamelCase__    ,		trust_remote_code=UpperCamelCase__						)
												self.assertEqual(reloaded_image_processor.__class__.__name__    ,		'''NewImageProcessor'''						)


						def        __A       (       self	:				Optional[Any]						):





												'''simple docstring'''





												try:
																		AutoConfig.register('''custom'''    ,		UpperCamelCase__						)
																		AutoImageProcessor.register(UpperCamelCase__    ,		UpperCamelCase__						)
																		# Trying to register something existing in the Transformers library will raise an error
																		with self.assertRaises(UpperCamelCase__						):
																								AutoImageProcessor.register(UpperCamelCase__    ,		UpperCamelCase__						)

																		with tempfile.TemporaryDirectory() as tmpdirname:
																								SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							Path(UpperCamelCase__						) / '''preprocessor_config.json'''
																								SCREAMING_SNAKE_CASE							:    Any							=							Path(UpperCamelCase__						) / '''config.json'''
																								json.dump(
																								    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}    ,		open(UpperCamelCase__    ,		'''w'''						)    ,		)
																								json.dump({'''model_type''': '''clip'''}    ,		open(UpperCamelCase__    ,		'''w'''						)						)

																								SCREAMING_SNAKE_CASE							:    Tuple							=							CustomImageProcessor.from_pretrained(UpperCamelCase__						)

																		# Now that the config is registered, it can be used as any other config with the auto-API
																		with tempfile.TemporaryDirectory() as tmp_dir:
																								image_processor.save_pretrained(UpperCamelCase__						)
																								SCREAMING_SNAKE_CASE							:    Any							=							AutoImageProcessor.from_pretrained(UpperCamelCase__						)
																								self.assertIsInstance(UpperCamelCase__    ,		UpperCamelCase__						)

												finally:
																		if "custom" in CONFIG_MAPPING._extra_content:
																								del CONFIG_MAPPING._extra_content["custom"]
																		if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																								del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]



						def        __A       (       self	:				Any						):





												'''simple docstring'''





												class 					lowercase__						(   UpperCamelCase_):
																		UpperCamelCase_     =		True

												try:
																		AutoConfig.register('''custom'''    ,		UpperCamelCase__						)
																		AutoImageProcessor.register(UpperCamelCase__    ,		UpperCamelCase__						)
																		# If remote code is not set, the default is to use local
																		SCREAMING_SNAKE_CASE							:    str							=							AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''						)
																		self.assertEqual(image_processor.__class__.__name__    ,		'''NewImageProcessor'''						)
																		self.assertTrue(image_processor.is_local						)

																		# If remote code is disabled, we load the local one.
																		SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							AutoImageProcessor.from_pretrained(
																		    '''hf-internal-testing/test_dynamic_image_processor'''    ,		trust_remote_code=UpperCamelCase__						)
																		self.assertEqual(image_processor.__class__.__name__    ,		'''NewImageProcessor'''						)
																		self.assertTrue(image_processor.is_local						)

																		# If remote is enabled, we load from the Hub
																		SCREAMING_SNAKE_CASE							:    Optional[Any]							=							AutoImageProcessor.from_pretrained(
																		    '''hf-internal-testing/test_dynamic_image_processor'''    ,		trust_remote_code=UpperCamelCase__						)
																		self.assertEqual(image_processor.__class__.__name__    ,		'''NewImageProcessor'''						)
																		self.assertTrue(not hasattr(UpperCamelCase__    ,		'''is_local'''						)						)

												finally:
																		if "custom" in CONFIG_MAPPING._extra_content:
																								del CONFIG_MAPPING._extra_content["custom"]
																		if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																								del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





