


import unittest

import numpy as np

from transformers import RobertaPreLayerNormConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
    import jax.numpy as jnp

    from transformers.models.roberta_prelayernorm.modeling_flax_roberta_prelayernorm import (
        FlaxRobertaPreLayerNormForCausalLM,
        FlaxRobertaPreLayerNormForMaskedLM,
        FlaxRobertaPreLayerNormForMultipleChoice,
        FlaxRobertaPreLayerNormForQuestionAnswering,
        FlaxRobertaPreLayerNormForSequenceClassification,
        FlaxRobertaPreLayerNormForTokenClassification,
        FlaxRobertaPreLayerNormModel,
    )







class 			_lowercase (							unittest.TestCase):


       """simple docstring"""




       def __init__(   self							:			Tuple					,      __lowerCamelCase							:			Union[str, Any]					,      __lowerCamelCase							:			List[Any]=13					,      __lowerCamelCase							:			Tuple=7					,      __lowerCamelCase							:			Dict=True					,      __lowerCamelCase							:			Any=True					,      __lowerCamelCase							:			Optional[int]=True					,      __lowerCamelCase							:			List[Any]=True					,      __lowerCamelCase							:			Tuple=99					,      __lowerCamelCase							:			Any=32					,      __lowerCamelCase							:			Dict=5					,      __lowerCamelCase							:			Union[str, Any]=4					,      __lowerCamelCase							:			List[str]=37					,      __lowerCamelCase							:			List[str]="gelu"					,      __lowerCamelCase							:			str=0.1					,      __lowerCamelCase							:			List[Any]=0.1					,      __lowerCamelCase							:			Union[str, Any]=512					,      __lowerCamelCase							:			Dict=16					,      __lowerCamelCase							:			List[Any]=2					,      __lowerCamelCase							:			Tuple=0.0_2					,      __lowerCamelCase							:			List[Any]=4					,      ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  parent
             lowerCamelCase__		:				Union[str, Any]    =  batch_size
             lowerCamelCase__		:				str    =  seq_length
             lowerCamelCase__		:				Tuple    =  is_training
             lowerCamelCase__		:				Dict    =  use_attention_mask
             lowerCamelCase__		:				Optional[int]    =  use_token_type_ids
             lowerCamelCase__		:				Dict    =  use_labels
             lowerCamelCase__		:				Any    =  vocab_size
             lowerCamelCase__		:				Any    =  hidden_size
             lowerCamelCase__		:				Dict    =  num_hidden_layers
             lowerCamelCase__		:				Optional[int]    =  num_attention_heads
             lowerCamelCase__		:				str    =  intermediate_size
             lowerCamelCase__		:				List[str]    =  hidden_act
             lowerCamelCase__		:				Tuple    =  hidden_dropout_prob
             lowerCamelCase__		:				List[str]    =  attention_probs_dropout_prob
             lowerCamelCase__		:				Union[str, Any]    =  max_position_embeddings
             lowerCamelCase__		:				Dict    =  type_vocab_size
             lowerCamelCase__		:				str    =  type_sequence_label_size
             lowerCamelCase__		:				List[str]    =  initializer_range
             lowerCamelCase__		:				Any    =  num_choices





       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  ids_tensor([self.batch_size, self.seq_length]					,      self.vocab_size       )

             lowerCamelCase__		:				List[str]    =  None
             if self.use_attention_mask:
                   lowerCamelCase__		:				Dict    =  random_attention_mask([self.batch_size, self.seq_length]       )

             lowerCamelCase__		:				List[str]    =  None
             if self.use_token_type_ids:
                   lowerCamelCase__		:				Optional[Any]    =  ids_tensor([self.batch_size, self.seq_length]					,      self.type_vocab_size       )

             lowerCamelCase__		:				Dict    =  RobertaPreLayerNormConfig(
                 vocab_size=self.vocab_size					,      hidden_size=self.hidden_size					,      num_hidden_layers=self.num_hidden_layers					,      num_attention_heads=self.num_attention_heads					,      intermediate_size=self.intermediate_size					,      hidden_act=self.hidden_act					,      hidden_dropout_prob=self.hidden_dropout_prob					,      attention_probs_dropout_prob=self.attention_probs_dropout_prob					,      max_position_embeddings=self.max_position_embeddings					,      type_vocab_size=self.type_vocab_size					,      is_decoder=__lowerCamelCase					,      initializer_range=self.initializer_range					,      )

             return config, input_ids, token_type_ids, attention_mask





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Dict    =  self.prepare_config_and_inputs()
             lowerCamelCase__    ,   lowerCamelCase__    ,   lowerCamelCase__    ,   lowerCamelCase__		:				Dict    =  config_and_inputs
             lowerCamelCase__		:				Optional[int]    =  {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
             return config, inputs_dict





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  self.prepare_config_and_inputs()
             lowerCamelCase__    ,   lowerCamelCase__    ,   lowerCamelCase__    ,   lowerCamelCase__		:				Optional[int]    =  config_and_inputs

             lowerCamelCase__		:				Tuple    =  True
             lowerCamelCase__		:				int    =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
             lowerCamelCase__		:				Any    =  ids_tensor([self.batch_size, self.seq_length]					,      vocab_size=2       )

             return (
                 config,
                 input_ids,
                 token_type_ids,
                 encoder_hidden_states,
                 encoder_attention_mask,
             )







@require_flax
# Copied from tests.models.roberta.test_modelling_flax_roberta.FlaxRobertaPreLayerNormModelTest with ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta-base->andreasmadsen/efficient_mlm_m0.40
class 			_lowercase (							lowercase__							,  unittest.TestCase):


       """simple docstring"""




       A__											=			True

       A__											=			(
           (
               FlaxRobertaPreLayerNormModel,
               FlaxRobertaPreLayerNormForCausalLM,
               FlaxRobertaPreLayerNormForMaskedLM,
               FlaxRobertaPreLayerNormForSequenceClassification,
               FlaxRobertaPreLayerNormForTokenClassification,
               FlaxRobertaPreLayerNormForMultipleChoice,
               FlaxRobertaPreLayerNormForQuestionAnswering,
           )
           if is_flax_available()
           else ()
       )





       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  FlaxRobertaPreLayerNormModelTester(self       )





       @slow
       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             for model_class_name in self.all_model_classes:
                   lowerCamelCase__		:				int    =  model_class_name.from_pretrained("andreasmadsen/efficient_mlm_m0.40"					,      from_pt=__lowerCamelCase       )
                   lowerCamelCase__		:				Any    =  model(np.ones((1, 1)       )       )
                   self.assertIsNotNone(__lowerCamelCase       )







@require_flax
class 			_lowercase (							unittest.TestCase):


       """simple docstring"""




       @slow
       def 						lowerCAmelCase     (   self							:			List[str]       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  FlaxRobertaPreLayerNormForMaskedLM.from_pretrained("andreasmadsen/efficient_mlm_m0.40"					,      from_pt=__lowerCamelCase       )

             lowerCamelCase__		:				Tuple    =  np.array([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]					,      dtype=jnp.intaa       )
             lowerCamelCase__		:				int    =  model(__lowerCamelCase       )[0]
             lowerCamelCase__		:				Any    =  [1, 11, 50265]
             self.assertEqual(list(output.shape       )					,      __lowerCamelCase       )
             # compare the actual values for a slice.
             lowerCamelCase__		:				str    =  np.array(
                 [[[4_0.4_8_8_0, 1_8.0_1_9_9, -5.2_3_6_7], [-1.8_8_7_7, -4.0_8_8_5, 1_0.7_0_8_5], [-2.2_6_1_3, -5.6_1_1_0, 7.2_6_6_5]]]					,      dtype=np.floataa       )
             self.assertTrue(np.allclose(output[:, :3, :3]					,      __lowerCamelCase					,      atol=1E-4       )       )





       @slow
       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Any    =  FlaxRobertaPreLayerNormModel.from_pretrained("andreasmadsen/efficient_mlm_m0.40"					,      from_pt=__lowerCamelCase       )

             lowerCamelCase__		:				Dict    =  np.array([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]					,      dtype=jnp.intaa       )
             lowerCamelCase__		:				Tuple    =  model(__lowerCamelCase       )[0]
             # compare the actual values for a slice.
             lowerCamelCase__		:				Tuple    =  np.array(
                 [[[0.0_2_0_8, -0.0_3_5_6, 0.0_2_3_7], [-0.1_5_6_9, -0.0_4_1_1, -0.2_6_2_6], [0.1_8_7_9, 0.0_1_2_5, -0.0_0_8_9]]]					,      dtype=np.floataa       )
             self.assertTrue(np.allclose(output[:, :3, :3]					,      __lowerCamelCase					,      atol=1E-4       )       )






from .constants import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    TORCH_LAUNCH_PARAMS,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
)
from .dataclasses import (
    BnbQuantizationConfig,
    ComputeEnvironment,
    CustomDtype,
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    DynamoBackend,
    FPaRecipeKwargs,
    FullyShardedDataParallelPlugin,
    GradientAccumulationPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    KwargsHandler,
    LoggerType,
    MegatronLMPlugin,
    PrecisionType,
    ProjectConfiguration,
    RNGType,
    SageMakerDistributedType,
    TensorInformation,
    TorchDynamoPlugin,
)
from .environment import get_int_from_env, parse_choice_from_env, parse_flag_from_env
from .imports import (
    get_ccl_version,
    is_abit_bnb_available,
    is_abit_bnb_available,
    is_aim_available,
    is_bfaa_available,
    is_bnb_available,
    is_botoa_available,
    is_ccl_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_fpa_available,
    is_ipex_available,
    is_megatron_lm_available,
    is_mlflow_available,
    is_mps_available,
    is_npu_available,
    is_rich_available,
    is_safetensors_available,
    is_sagemaker_available,
    is_tensorboard_available,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)
from .modeling import (
    check_device_map,
    check_tied_parameters_in_config,
    check_tied_parameters_on_same_device,
    compute_module_sizes,
    convert_file_size_to_int,
    dtype_byte_size,
    find_tied_parameters,
    get_balanced_memory,
    get_max_layer_size,
    get_max_memory,
    get_mixed_precision_context_manager,
    id_tensor_storage,
    infer_auto_device_map,
    load_checkpoint_in_model,
    load_offloaded_weights,
    load_state_dict,
    named_module_tensors,
    retie_parameters,
    set_module_tensor_to_device,
    shard_checkpoint,
)
from .offload import (
    OffloadedWeightsLoader,
    PrefixedDataset,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
    save_offload_index,
)
from .operations import (
    broadcast,
    broadcast_object_list,
    concatenate,
    convert_outputs_to_fpaa,
    convert_to_fpaa,
    find_batch_size,
    find_device,
    gather,
    gather_object,
    get_data_structure,
    honor_type,
    initialize_tensors,
    is_namedtuple,
    is_tensor_information,
    is_torch_tensor,
    listify,
    pad_across_processes,
    recursively_apply,
    reduce,
    send_to_device,
    slice_tensors,
)
from .versions import compare_versions, is_torch_version


if is_deepspeed_available():
    from .deepspeed import (
        DeepSpeedEngineWrapper,
        DeepSpeedOptimizerWrapper,
        DeepSpeedSchedulerWrapper,
        DummyOptim,
        DummyScheduler,
        HfDeepSpeedConfig,
    )

from .bnb import has_abit_bnb_layers, load_and_quantize_model
from .fsdp_utils import load_fsdp_model, load_fsdp_optimizer, save_fsdp_model, save_fsdp_optimizer
from .launch import (
    PrepareForLaunch,
    _filter_args,
    prepare_deepspeed_cmd_env,
    prepare_multi_gpu_env,
    prepare_sagemager_args_inputs,
    prepare_simple_launcher_cmd_env,
    prepare_tpu,
)
from .megatron_lm import (
    AbstractTrainStep,
    BertTrainStep,
    GPTTrainStep,
    MegatronEngine,
    MegatronLMDummyDataLoader,
    MegatronLMDummyScheduler,
    MegatronLMOptimizerWrapper,
    MegatronLMSchedulerWrapper,
    TaTrainStep,
    avg_losses_across_data_parallel_group,
    gather_across_data_parallel_groups,
)
from .megatron_lm import initialize as megatron_lm_initialize
from .megatron_lm import prepare_data_loader as megatron_lm_prepare_data_loader
from .megatron_lm import prepare_model as megatron_lm_prepare_model
from .megatron_lm import prepare_optimizer as megatron_lm_prepare_optimizer
from .megatron_lm import prepare_scheduler as megatron_lm_prepare_scheduler
from .memory import find_executable_batch_size, release_memory
from .other import (
    extract_model_from_parallel,
    get_pretty_name,
    is_port_in_use,
    merge_dicts,
    patch_environment,
    save,
    wait_for_everyone,
    write_basic_config,
)
from .random import set_seed, synchronize_rng_state, synchronize_rng_states
from .torch_xla import install_xla
from .tqdm import tqdm
from .transformer_engine import convert_model, has_transformer_engine_layers


