code: 
  
  
  '''simple docstring'''
  
  
  
  import collections
  import inspect
  import unittest
  
  from transformers import SwinConfig
  from transformers.testing_utils import require_torch, require_vision, slow, torch_device
  from transformers.utils import cached_property, is_torch_available, is_vision_available
  
  from ...test_backbone_common import BackboneTesterMixin
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  	import torch
  	from torch import nn
  
  	from transformers import SwinBackbone, SwinForImageClassification, SwinForMaskedImageModeling, SwinModel
  	from transformers.models.swin.modeling_swin import SWIN_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  if is_vision_available():
  	from PIL import Image
  
  	from transformers import AutoImageProcessor
  
  
  
  
  
  class 							__UpperCamelCase       :
  
  
  
  
  	def __init__(     self		,       __a		,       __a=13		,       __a=32		,       __a=2		,       __a=3		,       __a=16		,       __a=[1, 2, 1]		,       __a=[2, 2, 4]		,       __a=2		,       __a=2.0		,       __a=True		,       __a=0.0		,       __a=0.0		,       __a=0.1		,       __a="gelu"		,       __a=False		,       __a=True		,       __a=0.02		,       __a=1E-5		,       __a=True		,       __a=None		,       __a=True		,       __a=10		,       __a=8		,       __a=["stage1", "stage2"]		,       __a=[1, 2]		,       ):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Tuple      					=						parent
  						__a    : List[Any]      					=						batch_size
  						__a    : List[Any]      					=						image_size
  						__a    : List[Any]      					=						patch_size
  						__a    : Dict      					=						num_channels
  						__a    : List[Any]      					=						embed_dim
  						__a    : Dict      					=						depths
  						__a    : Any      					=						num_heads
  						__a    : Optional[Any]      					=						window_size
  						__a    : int      					=						mlp_ratio
  						__a    : List[str]      					=						qkv_bias
  						__a    : List[str]      					=						hidden_dropout_prob
  						__a    : Dict      					=						attention_probs_dropout_prob
  						__a    : Tuple      					=						drop_path_rate
  						__a    : Union[str, Any]      					=						hidden_act
  						__a    : str      					=						use_absolute_embeddings
  						__a    : Any      					=						patch_norm
  						__a    : Optional[int]      					=						layer_norm_eps
  						__a    : Tuple      					=						initializer_range
  						__a    : Union[str, Any]      					=						is_training
  						__a    : Optional[Any]      					=						scope
  						__a    : str      					=						use_labels
  						__a    : int      					=						type_sequence_label_size
  						__a    : Union[str, Any]      					=						encoder_stride
  						__a    : Any      					=						out_features
  						__a    : List[str]      					=						out_indices
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Tuple      					=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)
  
  						__a    : Tuple      					=						None
  						if self.use_labels:
  											__a    : List[str]      					=						ids_tensor([self.batch_size]		,       self.type_sequence_label_size							)
  
  						__a    : List[Any]      					=						self.get_config()
  
  						return config, pixel_values, labels
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						return SwinConfig(
  						    image_size=self.image_size		,       patch_size=self.patch_size		,       num_channels=self.num_channels		,       embed_dim=self.embed_dim		,       depths=self.depths		,       num_heads=self.num_heads		,       window_size=self.window_size		,       mlp_ratio=self.mlp_ratio		,       qkv_bias=self.qkv_bias		,       hidden_dropout_prob=self.hidden_dropout_prob		,       attention_probs_dropout_prob=self.attention_probs_dropout_prob		,       drop_path_rate=self.drop_path_rate		,       hidden_act=self.hidden_act		,       use_absolute_embeddings=self.use_absolute_embeddings		,       path_norm=self.patch_norm		,       layer_norm_eps=self.layer_norm_eps		,       initializer_range=self.initializer_range		,       encoder_stride=self.encoder_stride		,       out_features=self.out_features		,       out_indices=self.out_indices		,       )
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Any      					=						SwinModel(config=__a							)
  						model.to(__a							)
  						model.eval()
  						__a    : Any      					=						model(__a							)
  
  						__a    : Optional[Any]      					=						((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths							) - 1))
  						__a    : Dict      					=						int(config.embed_dim * 2 ** (len(config.depths							) - 1)							)
  
  						self.parent.assertEqual(result.last_hidden_state.shape		,       (self.batch_size, expected_seq_len, expected_dim)							)
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Dict      					=						SwinBackbone(config=__a							)
  						model.to(__a							)
  						model.eval()
  						__a    : Optional[Any]      					=						model(__a							)
  
  						# verify hidden states
  						self.parent.assertEqual(len(result.feature_maps							)		,       len(config.out_features							)							)
  						self.parent.assertListEqual(list(result.feature_maps[0].shape							)		,       [self.batch_size, model.channels[0], 16, 16]							)
  
  						# verify channels
  						self.parent.assertEqual(len(model.channels							)		,       len(config.out_features							)							)
  
  						# verify backbone works with out_features=None
  						__a    : Tuple      					=						None
  						__a    : List[Any]      					=						SwinBackbone(config=__a							)
  						model.to(__a							)
  						model.eval()
  						__a    : List[Any]      					=						model(__a							)
  
  						# verify feature maps
  						self.parent.assertEqual(len(result.feature_maps							)		,       1							)
  						self.parent.assertListEqual(list(result.feature_maps[0].shape							)		,       [self.batch_size, model.channels[-1], 4, 4]							)
  
  						# verify channels
  						self.parent.assertEqual(len(model.channels							)		,       1							)
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : int      					=						SwinForMaskedImageModeling(config=__a							)
  						model.to(__a							)
  						model.eval()
  						__a    : Dict      					=						model(__a							)
  						self.parent.assertEqual(
  						    result.logits.shape		,       (self.batch_size, self.num_channels, self.image_size, self.image_size)							)
  
  						# test greyscale images
  						__a    : Optional[int]      					=						1
  						__a    : Optional[Any]      					=						SwinForMaskedImageModeling(__a							)
  						model.to(__a							)
  						model.eval()
  
  						__a    : str      					=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
  						__a    : List[str]      					=						model(__a							)
  						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, 1, self.image_size, self.image_size)							)
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : List[Any]      					=						self.type_sequence_label_size
  						__a    : int      					=						SwinForImageClassification(__a							)
  						model.to(__a							)
  						model.eval()
  						__a    : List[str]      					=						model(__a		,       labels=__a							)
  						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.type_sequence_label_size)							)
  
  						# test greyscale images
  						__a    : List[Any]      					=						1
  						__a    : Optional[Any]      					=						SwinForImageClassification(__a							)
  						model.to(__a							)
  						model.eval()
  
  						__a    : Any      					=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
  						__a    : str      					=						model(__a							)
  						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.type_sequence_label_size)							)
  
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Optional[Any]      					=						self.prepare_config_and_inputs()
  						(
  						    (
  						    __a
  						)					,       (
  						    __a
  						)					,       (
  						    __a
  						)					,       
  						)    : Any      					=						config_and_inputs
  						__a    : Union[str, Any]      					=						{'pixel_values': pixel_values}
  						return config, inputs_dict
  
  
  
  
  
  @require_torch
  class 							__UpperCamelCase       (	lowerCAmelCase_	,   lowerCAmelCase_	,   unittest.TestCase      ):
  	A_			    =					(
  	    (
  	        SwinModel,
  	        SwinBackbone,
  	        SwinForImageClassification,
  	        SwinForMaskedImageModeling,
  	    )
  	    if is_torch_available()
  	    else ()
  	)
  	A_			    =					(
  	    {"feature-extraction": SwinModel, "image-classification": SwinForImageClassification}
  	    if is_torch_available()
  	    else {}
  	)
  	A_			    =					True
  
  	A_			    =					False
  	A_			    =					False
  	A_			    =					False
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : List[str]      					=						SwinModelTester(self							)
  						__a    : Dict      					=						ConfigTester(self		,       config_class=__a		,       embed_dim=37							)
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						self.create_and_test_config_common_properties()
  						self.config_tester.create_and_test_config_to_json_string()
  						self.config_tester.create_and_test_config_to_json_file()
  						self.config_tester.create_and_test_config_from_and_save_pretrained()
  						self.config_tester.create_and_test_config_with_num_labels()
  						self.config_tester.check_config_can_be_init_without_params()
  						self.config_tester.check_config_arguments_init()
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						return
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Dict      					=						self.model_tester.prepare_config_and_inputs()
  						self.model_tester.create_and_check_model(*__a							)
  
  
  
  
  	@unittest.skip(reason='Got `CUDA error: misaligned address` with PyTorch 2.0.0.'							)
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						pass
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						super().test_training_gradient_checkpointing()
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : int      					=						self.model_tester.prepare_config_and_inputs()
  						self.model_tester.create_and_check_backbone(*__a							)
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : str      					=						self.model_tester.prepare_config_and_inputs()
  						self.model_tester.create_and_check_for_masked_image_modeling(*__a							)
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Optional[Any]      					=						self.model_tester.prepare_config_and_inputs()
  						self.model_tester.create_and_check_for_image_classification(*__a							)
  
  
  
  
  	@unittest.skip(reason='Swin does not use inputs_embeds'							)
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						pass
  
  
  
  
  	@unittest.skip(reason='Swin Transformer does not use feedforward chunking'							)
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						pass
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a					,       __a    : Optional[int]      					=						self.model_tester.prepare_config_and_inputs_for_common()
  
  						for model_class in self.all_model_classes:
  											__a    : Optional[int]      					=						model_class(__a							)
  											self.assertIsInstance(model.get_input_embeddings()		,       (nn.Module)							)
  											__a    : Dict      					=						model.get_output_embeddings()
  											self.assertTrue(x is None or isinstance(__a		,       nn.Linear							)							)
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a					,       __a    : Optional[int]      					=						self.model_tester.prepare_config_and_inputs_for_common()
  
  						for model_class in self.all_model_classes:
  											__a    : Optional[Any]      					=						model_class(__a							)
  											__a    : List[str]      					=						inspect.signature(model.forward							)
  											# signature.parameters is an OrderedDict => so arg_names order is deterministic
  											__a    : Tuple      					=						[*signature.parameters.keys()]
  
  											__a    : List[Any]      					=						['pixel_values']
  											self.assertListEqual(arg_names[:1]		,       __a							)
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a					,       __a    : Tuple      					=						self.model_tester.prepare_config_and_inputs_for_common()
  						__a    : Dict      					=						True
  
  						for model_class in self.all_model_classes:
  											__a    : int      					=						True
  											__a    : List[Any]      					=						False
  											__a    : int      					=						True
  											__a    : List[str]      					=						model_class(__a							)
  											model.to(__a							)
  											model.eval()
  											with torch.no_grad():
  																__a    : List[str]      					=						model(**self._prepare_for_class(__a		,       __a							)							)
  											__a    : List[Any]      					=						outputs.attentions
  											__a    : Optional[int]      					=						len(self.model_tester.depths							)
  											self.assertEqual(len(__a							)		,       __a							)
  
  											# check that output_attentions also work using config
  											del inputs_dict["output_attentions"]
  											__a    : List[Any]      					=						True
  											__a    : str      					=						config.window_size**2
  											__a    : Optional[Any]      					=						model_class(__a							)
  											model.to(__a							)
  											model.eval()
  											with torch.no_grad():
  																__a    : Optional[int]      					=						model(**self._prepare_for_class(__a		,       __a							)							)
  											__a    : str      					=						outputs.attentions
  											self.assertEqual(len(__a							)		,       __a							)
  
  											self.assertListEqual(
  											    list(attentions[0].shape[-3:]							)		,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]		,       )
  											__a    : List[str]      					=						len(__a							)
  
  											# Check attention is always last and order is fine
  											__a    : int      					=						True
  											__a    : int      					=						True
  											__a    : Optional[int]      					=						model_class(__a							)
  											model.to(__a							)
  											model.eval()
  											with torch.no_grad():
  																__a    : Union[str, Any]      					=						model(**self._prepare_for_class(__a		,       __a							)							)
  
  											# also another +1 for reshaped_hidden_states
  											__a    : List[Any]      					=						1 if model_class.__name__ == 'SwinBackbone' else 2
  											self.assertEqual(out_len + added_hidden_states		,       len(__a							)							)
  
  											__a    : Tuple      					=						outputs.attentions
  
  											self.assertEqual(len(__a							)		,       __a							)
  
  											self.assertListEqual(
  											    list(self_attentions[0].shape[-3:]							)		,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]		,       )
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a		,       __a		,       __a		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : str      					=						model_class(__a							)
  						model.to(__a							)
  						model.eval()
  
  						with torch.no_grad():
  											__a    : int      					=						model(**self._prepare_for_class(__a		,       __a							)							)
  
  						__a    : Any      					=						outputs.hidden_states
  
  						__a    : Any      					=						getattr(
  						    self.model_tester		,       'expected_num_hidden_layers'		,       len(self.model_tester.depths							) + 1							)
  						self.assertEqual(len(__a							)		,       __a							)
  
  						# Swin has a different seq_length
  						__a    : List[Any]      					=						(
  						    config.patch_size
  						    if isinstance(config.patch_size		,       collections.abc.Iterable							)
  						    else (config.patch_size, config.patch_size)
  						)
  
  						__a    : Optional[Any]      					=						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
  
  						self.assertListEqual(
  						    list(hidden_states[0].shape[-2:]							)		,       [num_patches, self.model_tester.embed_dim]		,       )
  
  						if not model_class.__name__ == "SwinBackbone":
  											__a    : List[str]      					=						outputs.reshaped_hidden_states
  											self.assertEqual(len(__a							)		,       __a							)
  
  											__a					,       __a					,       __a					,       __a    : List[str]      					=						reshaped_hidden_states[0].shape
  											__a    : int      					=						(
  											    reshaped_hidden_states[0].view(__a		,       __a		,       height * width							).permute(0		,       2		,       1							)
  											)
  											self.assertListEqual(
  											    list(reshaped_hidden_states.shape[-2:]							)		,       [num_patches, self.model_tester.embed_dim]		,       )
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a					,       __a    : Optional[Any]      					=						self.model_tester.prepare_config_and_inputs_for_common()
  
  						__a    : List[Any]      					=						(
  						    self.model_tester.image_size
  						    if isinstance(self.model_tester.image_size		,       collections.abc.Iterable							)
  						    else (self.model_tester.image_size, self.model_tester.image_size)
  						)
  
  						for model_class in self.all_model_classes:
  											__a    : Union[str, Any]      					=						True
  											self.check_hidden_states_output(__a		,       __a		,       __a		,       __a							)
  
  											# check that output_hidden_states also work using config
  											del inputs_dict["output_hidden_states"]
  											__a    : int      					=						True
  
  											self.check_hidden_states_output(__a		,       __a		,       __a		,       __a							)
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a					,       __a    : Optional[int]      					=						self.model_tester.prepare_config_and_inputs_for_common()
  						__a    : Dict      					=						3
  
  						__a    : str      					=						(
  						    self.model_tester.image_size
  						    if isinstance(self.model_tester.image_size		,       collections.abc.Iterable							)
  						    else (self.model_tester.image_size, self.model_tester.image_size)
  						)
  						__a    : List[Any]      					=						(
  						    config.patch_size
  						    if isinstance(config.patch_size		,       collections.abc.Iterable							)
  						    else (config.patch_size, config.patch_size)
  						)
  
  						__a    : Dict      					=						image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
  						__a    : List[str]      					=						image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])
  
  						for model_class in self.all_model_classes:
  											__a    : str      					=						True
  											self.check_hidden_states_output(__a		,       __a		,       __a		,       (padded_height, padded_width)							)
  
  											# check that output_hidden_states also work using config
  											del inputs_dict["output_hidden_states"]
  											__a    : str      					=						True
  											self.check_hidden_states_output(__a		,       __a		,       __a		,       (padded_height, padded_width)							)
  
  
  
  
  	@slow
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						for model_name in SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  											__a    : List[str]      					=						SwinModel.from_pretrained(__a							)
  											self.assertIsNotNone(__a							)
  
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a					,       __a    : Union[str, Any]      					=						self.model_tester.prepare_config_and_inputs_for_common()
  
  						__a    : Any      					=						_config_zero_init(__a							)
  						for model_class in self.all_model_classes:
  											__a    : Optional[Any]      					=						model_class(config=__a							)
  											for name, param in model.named_parameters():
  																if "embeddings" not in name and param.requires_grad:
  																					self.assertIn(
  																					    ((param.data.mean() * 1E9).round() / 1E9).item()		,       [0.0, 1.0]		,       msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""		,       )
  
  
  
  
  
  @require_vision
  @require_torch
  class 							__UpperCamelCase       (	unittest.TestCase      ):
  
  
  
  
  	@cached_property
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						return (
  						    AutoImageProcessor.from_pretrained('microsoft/swin-tiny-patch4-window7-224'							)
  						    if is_vision_available()
  						    else None
  						)
  
  
  
  
  
  	@slow
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Union[str, Any]      					=						SwinForImageClassification.from_pretrained('microsoft/swin-tiny-patch4-window7-224'							).to(__a							)
  						__a    : List[str]      					=						self.default_image_processor
  
  						__a    : Optional[Any]      					=						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'							)
  						__a    : List[str]      					=						image_processor(images=__a		,       return_tensors='pt'							).to(__a							)
  
  						# forward pass
  						with torch.no_grad():
  											__a    : List[Any]      					=						model(**__a							)
  
  						# verify the logits
  						__a    : Union[str, Any]      					=						torch.Size((1, 1000)							)
  						self.assertEqual(outputs.logits.shape		,       __a							)
  						__a    : Any      					=						torch.tensor([-0.0948, -0.6454, -0.0921]							).to(__a							)
  						self.assertTrue(torch.allclose(outputs.logits[0, :3]		,       __a		,       atol=1E-4							)							)
  
  
  
  
  
  @require_torch
  class 							__UpperCamelCase       (	unittest.TestCase	,   lowerCAmelCase_      ):
  	A_			    =					(SwinBackbone,) if is_torch_available() else ()
  	A_			    =					SwinConfig
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : Optional[int]      					=						SwinModelTester(self							)
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available
  
  
  __lowercase			:		Any =			{'configuration_visual_bert': ['VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VisualBertConfig']}
  
  try:
  	if not is_torch_available():
  		raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  	pass
  else:
  	__lowercase			:		Optional[int] =			[
  	    'VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
  	    'VisualBertForMultipleChoice',
  	    'VisualBertForPreTraining',
  	    'VisualBertForQuestionAnswering',
  	    'VisualBertForRegionToPhraseAlignment',
  	    'VisualBertForVisualReasoning',
  	    'VisualBertLayer',
  	    'VisualBertModel',
  	    'VisualBertPreTrainedModel',
  	]
  
  
  if TYPE_CHECKING:
  	from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig
  
  	try:
  		if not is_torch_available():
  			raise OptionalDependencyNotAvailable()
  	except OptionalDependencyNotAvailable:
  		pass
  	else:
  		from .modeling_visual_bert import (
  		    VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
  		    VisualBertForMultipleChoice,
  		    VisualBertForPreTraining,
  		    VisualBertForQuestionAnswering,
  		    VisualBertForRegionToPhraseAlignment,
  		    VisualBertForVisualReasoning,
  		    VisualBertLayer,
  		    VisualBertModel,
  		    VisualBertPreTrainedModel,
  		)
  
  
  else:
  	import sys
  
  	__lowercase			:		Tuple =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
  
  
  
  