code: 
  
  
  
  
  
  
  import functools
  import math
  
  import flax.linen as nn
  import jax
  import jax.numpy as jnp
  
  
  
  
  
  
  
  def 			lowercase__  (				A_:	List[str]     ,			A_:	str     ,			A_:	List[str]     ,			A_:	Union[str, Any]     ,			A_:	str = 4096		)  ->							Dict:
  
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
         __UpperCAmelCase    							=key.shape[-3:]
         __UpperCAmelCase    							=value.shape[-1]
         __UpperCAmelCase    							=min(A_     ,			A_		)
         __UpperCAmelCase    							=query / jnp.sqrt(A_		)
  
         @functools.partial(jax.checkpoint     ,			prevent_cse=A_		)
         def summarize_chunk(A_:	Tuple     ,			A_:	Optional[Any]     ,			A_:	Dict		):
                __UpperCAmelCase    							=jnp.einsum("""...qhd,...khd->...qhk"""     ,			A_     ,			A_     ,			precision=A_		)
  
                __UpperCAmelCase    							=jnp.max(A_     ,			axis=-1     ,			keepdims=A_		)
                __UpperCAmelCase    							=jax.lax.stop_gradient(A_		)
                __UpperCAmelCase    							=jnp.exp(attn_weights - max_score		)
  
                __UpperCAmelCase    							=jnp.einsum("""...vhf,...qhv->...qhf"""     ,			A_     ,			A_     ,			precision=A_		)
                __UpperCAmelCase    							=jnp.einsum("""...qhk->...qh"""     ,			A_		)
  
                return (exp_values, exp_weights.sum(axis=-1		), max_score)
  
         def chunk_scanner(A_:	List[Any]		):
                # julienne key array
                __UpperCAmelCase    							=jax.lax.dynamic_slice(
                    operand=A_     ,			start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]     ,			slice_sizes=list(key.shape[:-3]		) + [key_chunk_size, num_heads, k_features]     ,			)
  
                # julienne value array
                __UpperCAmelCase    							=jax.lax.dynamic_slice(
                    operand=A_     ,			start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]     ,			slice_sizes=list(value.shape[:-3]		) + [key_chunk_size, num_heads, v_features]     ,			)
  
                return summarize_chunk(A_     ,			A_     ,			A_		)
  
         __UpperCAmelCase    							=jax.lax.map(f=A_     ,			xs=jnp.arange(0     ,			A_     ,			A_		)		)
  
         __UpperCAmelCase    							=jnp.max(A_     ,			axis=0     ,			keepdims=A_		)
         __UpperCAmelCase    							=jnp.exp(chunk_max - global_max		)
  
         chunk_values *= jnp.expand_dims(A_     ,			axis=-1		)
         chunk_weights *= max_diffs
  
         __UpperCAmelCase    							=chunk_values.sum(axis=0		)
         __UpperCAmelCase    							=jnp.expand_dims(A_     ,			-1		).sum(axis=0		)
  
         return all_values / all_weights
  
  
  
  
  
  
  
  def 			lowercase__  (				A_:	str     ,			A_:	Union[str, Any]     ,			A_:	Tuple     ,			A_:	str=jax.lax.Precision.HIGHEST     ,			A_:	Dict = 1024     ,			A_:	str = 4096		)  ->							int:
  
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
         __UpperCAmelCase    							=query.shape[-3:]
  
         def chunk_scanner(A_:	Tuple     ,			A_:	int		):
                # julienne query array
                __UpperCAmelCase    							=jax.lax.dynamic_slice(
                    operand=A_     ,			start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]     ,			slice_sizes=list(query.shape[:-3]		) + [min(A_     ,			A_		), num_heads, q_features]     ,			)
  
                return (
                    chunk_idx + query_chunk_size,  # unused ignore it
                    _query_chunk_attention(
                        query=A_     ,			key=A_     ,			value=A_     ,			precision=A_     ,			key_chunk_size=A_		),
                )
  
         __UpperCAmelCase    							=jax.lax.scan(
             f=A_     ,			init=0     ,			xs=A_     ,			length=math.ceil(num_q / query_chunk_size		)		)
  
         return jnp.concatenate(A_     ,			axis=-3		)  # fuse the chunked result back
  
  
  
  class 							_A (    nn.Module							):
      """simple docstring"""
  
      lowerCamelCase  :       Optional[int]   =   42
      lowerCamelCase  :       int   =   8
      lowerCamelCase  :       str   =   64
      lowerCamelCase  :       Optional[Any]   =   0.0
      lowerCamelCase  :       Tuple   =   False
      lowerCamelCase  :       Tuple   =   jnp.floataa
  
  
  
      def _a       (							self			:    Union[str, Any]			)			->       List[str]:
             __UpperCAmelCase    							=self.dim_head * self.heads
             __UpperCAmelCase    							=self.dim_head**-0.5
  
             # Weights were exported with old names {to_q, to_k, to_v, to_out}
             __UpperCAmelCase    							=nn.Dense(__A   ,    use_bias=__A   ,    dtype=self.dtype   ,    name="""to_q"""			)
             __UpperCAmelCase    							=nn.Dense(__A   ,    use_bias=__A   ,    dtype=self.dtype   ,    name="""to_k"""			)
             __UpperCAmelCase    							=nn.Dense(__A   ,    use_bias=__A   ,    dtype=self.dtype   ,    name="""to_v"""			)
  
             __UpperCAmelCase    							=nn.Dense(self.query_dim   ,    dtype=self.dtype   ,    name="""to_out_0"""			)
             __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)
  
  
  
      def _a       (							self			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Tuple			)			->       Optional[Any]:
             __UpperCAmelCase    							=tensor.shape
             __UpperCAmelCase    							=self.heads
             __UpperCAmelCase    							=tensor.reshape(__A   ,    __A   ,    __A   ,    dim // head_size			)
             __UpperCAmelCase    							=jnp.transpose(__A   ,    (0, 2, 1, 3)			)
             __UpperCAmelCase    							=tensor.reshape(batch_size * head_size   ,    __A   ,    dim // head_size			)
             return tensor
  
  
  
      def _a       (							self			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       str:
             __UpperCAmelCase    							=tensor.shape
             __UpperCAmelCase    							=self.heads
             __UpperCAmelCase    							=tensor.reshape(batch_size // head_size   ,    __A   ,    __A   ,    __A			)
             __UpperCAmelCase    							=jnp.transpose(__A   ,    (0, 2, 1, 3)			)
             __UpperCAmelCase    							=tensor.reshape(batch_size // head_size   ,    __A   ,    dim * head_size			)
             return tensor
  
  
  
      def __call__(							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    int=None   ,    __SCREAMING_SNAKE_CASE			:    List[Any]=True			)			->       List[str]:
             __UpperCAmelCase    							=hidden_states if context is None else context
  
             __UpperCAmelCase    							=self.query(__A			)
             __UpperCAmelCase    							=self.key(__A			)
             __UpperCAmelCase    							=self.value(__A			)
  
             __UpperCAmelCase    							=self.reshape_heads_to_batch_dim(__A			)
             __UpperCAmelCase    							=self.reshape_heads_to_batch_dim(__A			)
             __UpperCAmelCase    							=self.reshape_heads_to_batch_dim(__A			)
  
             if self.use_memory_efficient_attention:
                    __UpperCAmelCase    							=query_states.transpose(1   ,    0   ,    2			)
                    __UpperCAmelCase    							=key_states.transpose(1   ,    0   ,    2			)
                    __UpperCAmelCase    							=value_states.transpose(1   ,    0   ,    2			)
  
                    # this if statement create a chunk size for each layer of the unet
                    # the chunk size is equal to the query_length dimension of the deepest layer of the unet
  
                    __UpperCAmelCase    							=query_states.shape[-3]
                    if flatten_latent_dim % 64 == 0:
                           __UpperCAmelCase    							=int(flatten_latent_dim / 64			)
                    elif flatten_latent_dim % 16 == 0:
                           __UpperCAmelCase    							=int(flatten_latent_dim / 16			)
                    elif flatten_latent_dim % 4 == 0:
                           __UpperCAmelCase    							=int(flatten_latent_dim / 4			)
                    else:
                           __UpperCAmelCase    							=int(__A			)
  
                    __UpperCAmelCase    							=jax_memory_efficient_attention(
                        __A   ,    __A   ,    __A   ,    query_chunk_size=__A   ,    key_chunk_size=4096 * 4			)
  
                    __UpperCAmelCase    							=hidden_states.transpose(1   ,    0   ,    2			)
             else:
                    # compute attentions
                    __UpperCAmelCase    							=jnp.einsum("""b i d, b j d->b i j"""   ,    __A   ,    __A			)
                    __UpperCAmelCase    							=attention_scores * self.scale
                    __UpperCAmelCase    							=nn.softmax(__A   ,    axis=2			)
  
                    # attend to values
                    __UpperCAmelCase    							=jnp.einsum("""b i j, b j d -> b i d"""   ,    __A   ,    __A			)
  
             __UpperCAmelCase    							=self.reshape_batch_dim_to_heads(__A			)
             __UpperCAmelCase    							=self.proj_attn(__A			)
             return self.dropout_layer(__A   ,    deterministic=__A			)
  
  
  
  
  class 							_A (    nn.Module							):
      """simple docstring"""
  
      lowerCamelCase  :       List[Any]   =   42
      lowerCamelCase  :       int   =   42
      lowerCamelCase  :       Optional[Any]   =   42
      lowerCamelCase  :       Optional[int]   =   0.0
      lowerCamelCase  :       Union[str, Any]   =   False
      lowerCamelCase  :       Union[str, Any]   =   jnp.floataa
      lowerCamelCase  :       Optional[int]   =   False
  
  
  
      def _a       (							self			:    str			)			->       Union[str, Any]:
             __UpperCAmelCase    							=FlaxAttention(
                 self.dim   ,    self.n_heads   ,    self.d_head   ,    self.dropout   ,    self.use_memory_efficient_attention   ,    dtype=self.dtype			)
             # cross attention
             __UpperCAmelCase    							=FlaxAttention(
                 self.dim   ,    self.n_heads   ,    self.d_head   ,    self.dropout   ,    self.use_memory_efficient_attention   ,    dtype=self.dtype			)
             __UpperCAmelCase    							=FlaxFeedForward(dim=self.dim   ,    dropout=self.dropout   ,    dtype=self.dtype			)
             __UpperCAmelCase    							=nn.LayerNorm(epsilon=1e-5   ,    dtype=self.dtype			)
             __UpperCAmelCase    							=nn.LayerNorm(epsilon=1e-5   ,    dtype=self.dtype			)
             __UpperCAmelCase    							=nn.LayerNorm(epsilon=1e-5   ,    dtype=self.dtype			)
             __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)
  
  
  
      def __call__(							self			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]=True			)			->       int:
             __UpperCAmelCase    							=hidden_states
             if self.only_cross_attention:
                    __UpperCAmelCase    							=self.attna(self.norma(__A			)   ,    __A   ,    deterministic=__A			)
             else:
                    __UpperCAmelCase    							=self.attna(self.norma(__A			)   ,    deterministic=__A			)
             __UpperCAmelCase    							=hidden_states + residual
  
             # cross attention
             __UpperCAmelCase    							=hidden_states
             __UpperCAmelCase    							=self.attna(self.norma(__A			)   ,    __A   ,    deterministic=__A			)
             __UpperCAmelCase    							=hidden_states + residual
  
             # feed forward
             __UpperCAmelCase    							=hidden_states
             __UpperCAmelCase    							=self.ff(self.norma(__A			)   ,    deterministic=__A			)
             __UpperCAmelCase    							=hidden_states + residual
  
             return self.dropout_layer(__A   ,    deterministic=__A			)
  
  
  
  
  class 							_A (    nn.Module							):
      """simple docstring"""
  
      lowerCamelCase  :       Tuple   =   42
      lowerCamelCase  :       str   =   42
      lowerCamelCase  :       Any   =   42
      lowerCamelCase  :       List[str]   =   1
      lowerCamelCase  :       Optional[Any]   =   0.0
      lowerCamelCase  :       Dict   =   False
      lowerCamelCase  :       Any   =   False
      lowerCamelCase  :       Union[str, Any]   =   jnp.floataa
      lowerCamelCase  :       int   =   False
  
  
  
      def _a       (							self			:    str			)			->       str:
             __UpperCAmelCase    							=nn.GroupNorm(num_groups=32   ,    epsilon=1e-5			)
  
             __UpperCAmelCase    							=self.n_heads * self.d_head
             if self.use_linear_projection:
                    __UpperCAmelCase    							=nn.Dense(__A   ,    dtype=self.dtype			)
             else:
                    __UpperCAmelCase    							=nn.Conv(
                        __A   ,    kernel_size=(1, 1)   ,    strides=(1, 1)   ,    padding="""VALID"""   ,    dtype=self.dtype   ,    )
  
             __UpperCAmelCase    							=[
                 FlaxBasicTransformerBlock(
                     __A   ,    self.n_heads   ,    self.d_head   ,    dropout=self.dropout   ,    only_cross_attention=self.only_cross_attention   ,    dtype=self.dtype   ,    use_memory_efficient_attention=self.use_memory_efficient_attention   ,    )
                 for _ in range(self.depth			)
             ]
  
             if self.use_linear_projection:
                    __UpperCAmelCase    							=nn.Dense(__A   ,    dtype=self.dtype			)
             else:
                    __UpperCAmelCase    							=nn.Conv(
                        __A   ,    kernel_size=(1, 1)   ,    strides=(1, 1)   ,    padding="""VALID"""   ,    dtype=self.dtype   ,    )
  
             __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)
  
  
  
      def __call__(							self			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    int   ,    __SCREAMING_SNAKE_CASE			:    List[str]=True			)			->       Optional[Any]:
             __UpperCAmelCase    							=hidden_states.shape
             __UpperCAmelCase    							=hidden_states
             __UpperCAmelCase    							=self.norm(__A			)
             if self.use_linear_projection:
                    __UpperCAmelCase    							=hidden_states.reshape(__A   ,    height * width   ,    __A			)
                    __UpperCAmelCase    							=self.proj_in(__A			)
             else:
                    __UpperCAmelCase    							=self.proj_in(__A			)
                    __UpperCAmelCase    							=hidden_states.reshape(__A   ,    height * width   ,    __A			)
  
             for transformer_block in self.transformer_blocks:
                    __UpperCAmelCase    							=transformer_block(__A   ,    __A   ,    deterministic=__A			)
  
             if self.use_linear_projection:
                    __UpperCAmelCase    							=self.proj_out(__A			)
                    __UpperCAmelCase    							=hidden_states.reshape(__A   ,    __A   ,    __A   ,    __A			)
             else:
                    __UpperCAmelCase    							=hidden_states.reshape(__A   ,    __A   ,    __A   ,    __A			)
                    __UpperCAmelCase    							=self.proj_out(__A			)
  
             __UpperCAmelCase    							=hidden_states + residual
             return self.dropout_layer(__A   ,    deterministic=__A			)
  
  
  
  
  class 							_A (    nn.Module							):
      """simple docstring"""
  
      lowerCamelCase  :       List[Any]   =   42
      lowerCamelCase  :       List[str]   =   0.0
      lowerCamelCase  :       str   =   jnp.floataa
  
  
  
      def _a       (							self			:    List[Any]			)			->       int:
             __UpperCAmelCase    							=FlaxGEGLU(self.dim   ,    self.dropout   ,    self.dtype			)
             __UpperCAmelCase    							=nn.Dense(self.dim   ,    dtype=self.dtype			)
  
  
  
      def __call__(							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    str=True			)			->       Tuple:
             __UpperCAmelCase    							=self.net_a(__A   ,    deterministic=__A			)
             __UpperCAmelCase    							=self.net_a(__A			)
             return hidden_states
  
  
  
  
  class 							_A (    nn.Module							):
      """simple docstring"""
  
      lowerCamelCase  :       Union[str, Any]   =   42
      lowerCamelCase  :       List[Any]   =   0.0
      lowerCamelCase  :       Union[str, Any]   =   jnp.floataa
  
  
  
      def _a       (							self			:    Tuple			)			->       List[Any]:
             __UpperCAmelCase    							=self.dim * 4
             __UpperCAmelCase    							=nn.Dense(inner_dim * 2   ,    dtype=self.dtype			)
             __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)
  
  
  
      def __call__(							self			:    str   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    str=True			)			->       Optional[int]:
             __UpperCAmelCase    							=self.proj(__A			)
             __UpperCAmelCase    							=jnp.split(__A   ,    2   ,    axis=2			)
             return self.dropout_layer(hidden_linear * nn.gelu(__A			)   ,    deterministic=__A			)
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  import unittest
  
  from transformers import ErnieMConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
     import torch
  
     from transformers import (
         ErnieMForInformationExtraction,
         ErnieMForMultipleChoice,
         ErnieMForQuestionAnswering,
         ErnieMForSequenceClassification,
         ErnieMForTokenClassification,
         ErnieMModel,
     )
     from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  class 							_A :
      """simple docstring"""
  
      def __init__(							self			:    Any   ,    __SCREAMING_SNAKE_CASE			:    str   ,    __SCREAMING_SNAKE_CASE			:    List[str]=13   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]=7   ,    __SCREAMING_SNAKE_CASE			:    List[Any]=True   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]=True   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]=True   ,    __SCREAMING_SNAKE_CASE			:    str=99   ,    __SCREAMING_SNAKE_CASE			:    int=32   ,    __SCREAMING_SNAKE_CASE			:    str=5   ,    __SCREAMING_SNAKE_CASE			:    int=4   ,    __SCREAMING_SNAKE_CASE			:    List[str]=37   ,    __SCREAMING_SNAKE_CASE			:    int="gelu"   ,    __SCREAMING_SNAKE_CASE			:    Tuple=0.1   ,    __SCREAMING_SNAKE_CASE			:    List[str]=0.1   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]=512   ,    __SCREAMING_SNAKE_CASE			:    int=16   ,    __SCREAMING_SNAKE_CASE			:    int=2   ,    __SCREAMING_SNAKE_CASE			:    Any=0.02   ,    __SCREAMING_SNAKE_CASE			:    Dict=3   ,    __SCREAMING_SNAKE_CASE			:    Dict=4   ,    __SCREAMING_SNAKE_CASE			:    str=None   ,    )			->       Optional[Any]:
             __UpperCAmelCase    							=parent
             __UpperCAmelCase    							=batch_size
             __UpperCAmelCase    							=seq_length
             __UpperCAmelCase    							=is_training
             __UpperCAmelCase    							=use_input_mask
             __UpperCAmelCase    							=use_labels
             __UpperCAmelCase    							=vocab_size
             __UpperCAmelCase    							=hidden_size
             __UpperCAmelCase    							=num_hidden_layers
             __UpperCAmelCase    							=num_attention_heads
             __UpperCAmelCase    							=intermediate_size
             __UpperCAmelCase    							=hidden_act
             __UpperCAmelCase    							=hidden_dropout_prob
             __UpperCAmelCase    							=attention_probs_dropout_prob
             __UpperCAmelCase    							=max_position_embeddings
             __UpperCAmelCase    							=type_vocab_size
             __UpperCAmelCase    							=type_sequence_label_size
             __UpperCAmelCase    							=initializer_range
             __UpperCAmelCase    							=num_labels
             __UpperCAmelCase    							=num_choices
             __UpperCAmelCase    							=scope
  
  
  
      def _a       (							self			:    str			)			->       Optional[Any]:
             __UpperCAmelCase    							=ids_tensor([self.batch_size, self.seq_length]   ,    self.vocab_size			)
  
             __UpperCAmelCase    							=None
             if self.use_input_mask:
                    __UpperCAmelCase    							=random_attention_mask([self.batch_size, self.seq_length]			)
  
             __UpperCAmelCase    							=None
             __UpperCAmelCase    							=None
             __UpperCAmelCase    							=None
             if self.use_labels:
                    __UpperCAmelCase    							=ids_tensor([self.batch_size]   ,    self.type_sequence_label_size			)
                    __UpperCAmelCase    							=ids_tensor([self.batch_size, self.seq_length]   ,    self.num_labels			)
                    __UpperCAmelCase    							=ids_tensor([self.batch_size]   ,    self.num_choices			)
  
             __UpperCAmelCase    							=self.get_config()
  
             return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
      def _a       (							self			:    Optional[Any]			)			->       Any:
             __UpperCAmelCase    							=ids_tensor([self.batch_size, self.seq_length]   ,    self.vocab_size			)
  
             __UpperCAmelCase    							=None
             if self.use_input_mask:
                    __UpperCAmelCase    							=random_attention_mask([self.batch_size, self.seq_length]			)
             __UpperCAmelCase    							=self.get_config()
  
             return config, input_ids, input_mask
  
  
  
      def _a       (							self			:    List[str]			)			->       List[str]:
             return ErnieMConfig(
                 vocab_size=self.vocab_size   ,    hidden_size=self.hidden_size   ,    num_hidden_layers=self.num_hidden_layers   ,    num_attention_heads=self.num_attention_heads   ,    intermediate_size=self.intermediate_size   ,    hidden_act=self.hidden_act   ,    hidden_dropout_prob=self.hidden_dropout_prob   ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,    max_position_embeddings=self.max_position_embeddings   ,    type_vocab_size=self.type_vocab_size   ,    initializer_range=self.initializer_range   ,    )
  
  
  
      def _a       (							self			:    str   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    str   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       Optional[int]:
             __UpperCAmelCase    							=ErnieMModel(config=__SCREAMING_SNAKE_CASE			)
             model.to(__SCREAMING_SNAKE_CASE			)
             model.eval()
             __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE   ,    return_dict=__SCREAMING_SNAKE_CASE			)
             self.parent.assertEqual(result.last_hidden_state.shape   ,    (self.batch_size, self.seq_length, self.hidden_size)			)
  
  
  
      def _a       (							self			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       int:
             __UpperCAmelCase    							=ErnieMForQuestionAnswering(config=__SCREAMING_SNAKE_CASE			)
             model.to(__SCREAMING_SNAKE_CASE			)
             model.eval()
             __UpperCAmelCase    							=model(
                 __SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    start_positions=__SCREAMING_SNAKE_CASE   ,    end_positions=__SCREAMING_SNAKE_CASE   ,    )
             self.parent.assertEqual(result.start_logits.shape   ,    (self.batch_size, self.seq_length)			)
             self.parent.assertEqual(result.end_logits.shape   ,    (self.batch_size, self.seq_length)			)
  
  
  
      def _a       (							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       Any:
             __UpperCAmelCase    							=ErnieMForInformationExtraction(config=__SCREAMING_SNAKE_CASE			)
             model.to(__SCREAMING_SNAKE_CASE			)
             model.eval()
             __UpperCAmelCase    							=torch.ones_like(__SCREAMING_SNAKE_CASE   ,    dtype=torch.floataa			)
             __UpperCAmelCase    							=model(
                 __SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    start_positions=__SCREAMING_SNAKE_CASE   ,    end_positions=__SCREAMING_SNAKE_CASE   ,    )
             self.parent.assertEqual(result.start_logits.shape   ,    (self.batch_size, self.seq_length)			)
             self.parent.assertEqual(result.end_logits.shape   ,    (self.batch_size, self.seq_length)			)
  
  
  
      def _a       (							self			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       str:
             __UpperCAmelCase    							=self.num_labels
             __UpperCAmelCase    							=ErnieMForSequenceClassification(__SCREAMING_SNAKE_CASE			)
             model.to(__SCREAMING_SNAKE_CASE			)
             model.eval()
             __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    labels=__SCREAMING_SNAKE_CASE			)
             self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_labels)			)
  
  
  
      def _a       (							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       Dict:
             __UpperCAmelCase    							=self.num_labels
             __UpperCAmelCase    							=ErnieMForTokenClassification(config=__SCREAMING_SNAKE_CASE			)
             model.to(__SCREAMING_SNAKE_CASE			)
             model.eval()
             input_ids.to(__SCREAMING_SNAKE_CASE			)
             input_mask.to(__SCREAMING_SNAKE_CASE			)
             token_labels.to(__SCREAMING_SNAKE_CASE			)
  
             __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    labels=__SCREAMING_SNAKE_CASE			)
  
             self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.num_labels)			)
  
  
  
      def _a       (							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    int   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       List[Any]:
             __UpperCAmelCase    							=self.num_choices
             __UpperCAmelCase    							=ErnieMForMultipleChoice(config=__SCREAMING_SNAKE_CASE			)
             model.to(__SCREAMING_SNAKE_CASE			)
             model.eval()
             __UpperCAmelCase    							=input_ids.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
             __UpperCAmelCase    							=input_mask.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
             __UpperCAmelCase    							=model(
                 __SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    labels=__SCREAMING_SNAKE_CASE   ,    )
             self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_choices)			)
  
  
  
      def _a       (							self			:    Any			)			->       List[str]:
             __UpperCAmelCase    							=self.prepare_config_and_inputs()
             (
                 (
                 __UpperCAmelCase
             )							,   (
                 __UpperCAmelCase
             )							,   (
                 __UpperCAmelCase
             )							,   (
                 __UpperCAmelCase
             )							,   (
                 __UpperCAmelCase
             )							,   (
                 __UpperCAmelCase
             )							,   
             )    							=config_and_inputs
             __UpperCAmelCase    							={"""input_ids""": input_ids, """attention_mask""": input_mask}
             return config, inputs_dict
  
  
  
  
  @require_torch
  class 							_A (    UpperCamelCase				,				UpperCamelCase				,				unittest.TestCase							):
      """simple docstring"""
  
      lowerCamelCase  :       Union[str, Any]   =   (
          (
              ErnieMModel,
              ErnieMForMultipleChoice,
              ErnieMForQuestionAnswering,
              ErnieMForSequenceClassification,
              ErnieMForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      lowerCamelCase  :       Optional[Any]   =   ()
      lowerCamelCase  :       str   =   (
          {
              'feature-extraction': ErnieMModel,
              'question-answering': ErnieMForQuestionAnswering,
              'text-classification': ErnieMForSequenceClassification,
              'token-classification': ErnieMForTokenClassification,
              'zero-shot': ErnieMForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      lowerCamelCase  :       Optional[int]   =   False
  
  
  
      def _a       (							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    int   ,    __SCREAMING_SNAKE_CASE			:    int			)			->       Optional[int]:
             if pipeline_test_casse_name == "QAPipelineTests":
                    return True
  
             return False
  
  
  
      def _a       (							self			:    Optional[int]			)			->       List[Any]:
             __UpperCAmelCase    							=ErnieMModelTester(self			)
             __UpperCAmelCase    							=ConfigTester(self   ,    config_class=__SCREAMING_SNAKE_CASE   ,    hidden_size=37			)
  
  
  
      def _a       (							self			:    int			)			->       List[str]:
             self.config_tester.run_common_tests()
  
  
  
      def _a       (							self			:    str			)			->       str:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE			)
  
  
  
      def _a       (							self			:    Optional[int]			)			->       List[str]:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                    __UpperCAmelCase    							=type
                    self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE			)
  
  
  
      def _a       (							self			:    Optional[int]			)			->       str:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE			)
  
  
  
      def _a       (							self			:    List[str]			)			->       Union[str, Any]:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE			)
  
  
  
      def _a       (							self			:    Any			)			->       str:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE			)
  
  
  
      def _a       (							self			:    str			)			->       List[Any]:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_information_extraction(*__SCREAMING_SNAKE_CASE			)
  
  
  
      def _a       (							self			:    List[Any]			)			->       List[str]:
             __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE			)
  
  
  
      @slow
      def _a       (							self			:    List[str]			)			->       Optional[Any]:
             for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    __UpperCAmelCase    							=ErnieMModel.from_pretrained(__SCREAMING_SNAKE_CASE			)
                    self.assertIsNotNone(__SCREAMING_SNAKE_CASE			)
  
  
  
  
  @require_torch
  class 							_A (    unittest.TestCase							):
      """simple docstring"""
  
      @slow
      def _a       (							self			:    Tuple			)			->       Optional[Any]:
             __UpperCAmelCase    							=ErnieMModel.from_pretrained("""susnato/ernie-m-base_pytorch"""			)
             model.eval()
             __UpperCAmelCase    							=torch.tensor([[0, 1, 2, 3, 4, 5]]			)
             __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE			)[0]
  
             # TODO Replace vocab size
             __UpperCAmelCase    							=768
  
             __UpperCAmelCase    							=torch.Size((1, 6, hidden_size)			)
             self.assertEqual(output.shape   ,    __SCREAMING_SNAKE_CASE			)
  
             __UpperCAmelCase    							=torch.tensor(
                 [[[-0.0_012, 0.1_245, -0.0_214], [-0.0_742, 0.0_244, -0.0_771], [-0.0_333, 0.1_164, -0.1_554]]]			)
  
             self.assertTrue(torch.allclose(output[:, :3, :3]   ,    __SCREAMING_SNAKE_CASE   ,    atol=1e-3			)			)
  
  
  
  
  