code: 
  
  
  """simple docstring"""
  
  
  
  
  
  
  
  import math
  from collections import OrderedDict
  
  import torch
  from packaging import version
  from torch import Tensor, nn
  
  from .utils import logging
  
  
  lowerCamelCase_     :     List[Any]    		=						logging.get_logger(__name__)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def __init__(	self		)      ->  List[str]:
       super().__init__()
       if version.parse(torch.__version__		) < version.parse('''1.12.0'''		):
           raise ImportError(
               f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
               '''PytorchGELUTanh. Please upgrade torch.'''		)
  
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return nn.functional.gelu(__A				,					approximate='''tanh'''		)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi		) * (input + 0.044_715 * torch.pow(__A				,					3.0		))		))
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def __init__(	self				,					__A = False		)      ->  List[Any]:
       super().__init__()
       if use_gelu_python:
           a					=self._gelu_python
       else:
           a					=nn.functional.gelu
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0		)		))
  
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return self.act(__A		)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)		))
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return input * torch.sigmoid(1.702 * input		)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def __init__(	self				,					__A				,					__A		)      ->  Optional[Any]:
       if min > max:
           raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''		)
  
       super().__init__()
       a					=min
       a					=max
  
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return torch.clip(gelu(__A		)				,					self.min				,					self.max		)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def __init__(	self		)      ->  List[Any]:
       super().__init__()
       a					=math.sqrt(2 / math.pi		)
  
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(__A				,					3		))		))
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return nn.functional.silu(__A		)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def __init__(	self		)      ->  Optional[Any]:
       super().__init__()
       if version.parse(torch.__version__		) < version.parse('''1.9.0'''		):
           a					=self._mish_python
       else:
           a					=nn.functional.mish
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return input * torch.tanh(nn.functional.softplus(__A		)		)
  
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return self.act(__A		)
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  Tensor:
       return input
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A				,					__A=0.707_107				,					__A=0.282_095		)      ->  List[Any]:
       a					=(input - mu).div(sigma * math.sqrt(2.0		)		)
       return 0.5 * (1.0 + torch.erf(__A		))
  
  
  
  
  
  
  class    __A       ( nn.Module    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def    SCREAMING_SNAKE_CASE							(	self				,					__A		)      ->  List[str]:
       a					=nn.functional.relu(__A		)
       a					=torch.square(__A		)
       return squared
  
  
  
  
  
  
  class    __A       ( _SCREAMING_SNAKE_CASE    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   def __getitem__(	self				,					__A		)      ->  List[str]:
       a					=super().__getitem__(__A		)
       a	,   a					=content if isinstance(__A				,					__A		) else (content, {})
       return cls(**__A		)
  
  lowerCamelCase_     :     List[str]    		=						{
      """gelu""": GELUActivation,
      """gelu_10""": (ClippedGELUActivation, {"""min""": -1_0, """max""": 1_0}),
      """gelu_fast""": FastGELUActivation,
      """gelu_new""": NewGELUActivation,
      """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
      """gelu_pytorch_tanh""": PytorchGELUTanh,
      """gelu_accurate""": AccurateGELUActivation,
      """laplace""": LaplaceActivation,
      """linear""": LinearActivation,
      """mish""": MishActivation,
      """quick_gelu""": QuickGELUActivation,
      """relu""": nn.ReLU,
      """relu2""": ReLUSquaredActivation,
      """relu6""": nn.ReLUa,
      """sigmoid""": nn.Sigmoid,
      """silu""": SiLUActivation,
      """swish""": SiLUActivation,
      """tanh""": nn.Tanh,
  }
  lowerCamelCase_     :     Optional[int]    		=						ClassInstantier(ACTaCLS)
  def _A		(		lowercase	):
  
  
  
  
  
  
  
      """simple docstring"""
      if activation_string in ACTaFN:
          return ACTaFN[activation_string]
      else:
          raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()	)}'''	)
  
  
  # For backwards compatibility with: from activations import gelu_python
  lowerCamelCase_     :     Dict    		=						get_activation("""gelu_python""")
  lowerCamelCase_     :     Optional[Any]    		=						get_activation("""gelu_new""")
  lowerCamelCase_     :     Dict    		=						get_activation("""gelu""")
  lowerCamelCase_     :     str    		=						get_activation("""gelu_fast""")
  lowerCamelCase_     :     Any    		=						get_activation("""quick_gelu""")
  lowerCamelCase_     :     Dict    		=						get_activation("""silu""")
  lowerCamelCase_     :     List[Any]    		=						get_activation("""mish""")
  
  
  
  
  
  lowerCamelCase_     :     Union[str, Any]    		=						get_activation("""linear""")
style_context: 
  
  
  """simple docstring"""
  
  
  
  
  
  
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  lowerCamelCase_     :     int    		=						logging.get_logger(__name__)
  
  lowerCamelCase_     :     Optional[Any]    		=						{
      """dandelin/vilt-b32-mlm""": """https://huggingface.co/dandelin/vilt-b32-mlm/blob/main/config.json"""
  }
  
  
  
  
  
  
  class    __A       ( _SCREAMING_SNAKE_CASE    ):
  
  
  
  
  
  
   """simple docstring"""
  
  
  
  
   __lowerCAmelCase				      =						"vilt"
  
  
   def __init__(	self				,					__A=3_0522				,					__A=2				,					__A=2				,					__A=40				,					__A=768				,					__A=12				,					__A=12				,					__A=3072				,					__A="gelu"				,					__A=0.0				,					__A=0.0				,					__A=0.02				,					__A=1E-1_2				,					__A=384				,					__A=32				,					__A=3				,					__A=True				,					__A=-1				,					__A=False				,					__A=-1				,					**__A				,					)      ->  Dict:
       super().__init__(tie_word_embeddings=__A				,					**__A		)
  
       a					=vocab_size
       a					=type_vocab_size
       a					=modality_type_vocab_size
       a					=max_position_embeddings
  
       a					=hidden_size
       a					=num_hidden_layers
       a					=num_attention_heads
       a					=intermediate_size
       a					=hidden_act
       a					=hidden_dropout_prob
       a					=attention_probs_dropout_prob
       a					=initializer_range
       a					=layer_norm_eps
  
       a					=image_size
       a					=patch_size
       a					=num_channels
       a					=qkv_bias
       a					=max_image_length
       a					=num_images