




"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowercase	      =    logging.get_logger(__name__)

_lowercase	      =    {
    """google/mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/config.json"""
}

class       __lowercase       (			snake_case_	):


				'''simple docstring'''


				_lowerCAmelCase				=MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
				_lowerCAmelCase				="mobilebert"

				def __init__(   self   :     Optional[Any]      ,SCREAMING_SNAKE_CASE_   :     Any=3_0_5_2_2      ,SCREAMING_SNAKE_CASE_   :     Dict=5_1_2      ,SCREAMING_SNAKE_CASE_   :     List[Any]=2_4      ,SCREAMING_SNAKE_CASE_   :     Optional[int]=4      ,SCREAMING_SNAKE_CASE_   :     Dict=5_1_2      ,SCREAMING_SNAKE_CASE_   :     Any="relu"      ,SCREAMING_SNAKE_CASE_   :     str=0.0      ,SCREAMING_SNAKE_CASE_   :     List[Any]=0.1      ,SCREAMING_SNAKE_CASE_   :     Optional[int]=5_1_2      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=2      ,SCREAMING_SNAKE_CASE_   :     Dict=0.02      ,SCREAMING_SNAKE_CASE_   :     List[str]=1E-12      ,SCREAMING_SNAKE_CASE_   :     Optional[Any]=0      ,SCREAMING_SNAKE_CASE_   :     Optional[int]=1_2_8      ,SCREAMING_SNAKE_CASE_   :     Any=True      ,SCREAMING_SNAKE_CASE_   :     int=True      ,SCREAMING_SNAKE_CASE_   :     str=1_2_8      ,SCREAMING_SNAKE_CASE_   :     List[str]=False      ,SCREAMING_SNAKE_CASE_   :     int=True      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=4      ,SCREAMING_SNAKE_CASE_   :     List[Any]="no_norm"      ,SCREAMING_SNAKE_CASE_   :     List[Any]=True      ,SCREAMING_SNAKE_CASE_   :     Dict=None      ,**SCREAMING_SNAKE_CASE_   :     Optional[Any]      ,):
										super().__init__(pad_token_id=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      )

										snake_case__       :					List[str]										=					vocab_size
										snake_case__       :					Optional[int]										=					hidden_size
										snake_case__       :					Dict										=					num_hidden_layers
										snake_case__       :					Any										=					num_attention_heads
										snake_case__       :					int										=					hidden_act
										snake_case__       :					Optional[int]										=					intermediate_size
										snake_case__       :					Dict										=					hidden_dropout_prob
										snake_case__       :					Dict										=					attention_probs_dropout_prob
										snake_case__       :					str										=					max_position_embeddings
										snake_case__       :					Optional[int]										=					type_vocab_size
										snake_case__       :					List[str]										=					initializer_range
										snake_case__       :					str										=					layer_norm_eps
										snake_case__       :					str										=					embedding_size
										snake_case__       :					Dict										=					trigram_input
										snake_case__       :					Tuple										=					use_bottleneck
										snake_case__       :					Optional[int]										=					intra_bottleneck_size
										snake_case__       :					List[Any]										=					use_bottleneck_attention
										snake_case__       :					int										=					key_query_shared_bottleneck
										snake_case__       :					List[str]										=					num_feedforward_networks
										snake_case__       :					Union[str, Any]										=					normalization_type
										snake_case__       :					Tuple										=					classifier_activation

										if self.use_bottleneck:
																snake_case__       :					Optional[int]										=					intra_bottleneck_size
										else:
																snake_case__       :					Tuple										=					hidden_size

										snake_case__       :					Optional[Any]										=					classifier_dropout







class       __lowercase       (			snake_case_	):


				'''simple docstring'''


				@property
				def       _lowerCamelCase					(   self   :     Any      ):
										if self.task == "multiple-choice":
																snake_case__       :					Any										=					{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
										else:
																snake_case__       :					List[str]										=					{0: '''batch''', 1: '''sequence'''}
										return OrderedDict(
										    [
										        ('''input_ids''', dynamic_axis),
										        ('''attention_mask''', dynamic_axis),
										        ('''token_type_ids''', dynamic_axis),
										    ]      )








"""simple docstring"""





import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


_lowercase	      =    get_logger(__name__)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase = None      ,			):
						'''simple docstring'''


						for i, state in enumerate(lowerCamelCase		):
												snake_case__       :					List[Any]										=					f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
												snake_case__       :					Tuple										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Model weights saved in {output_model_file}'''		)
						# Optimizer states
						for i, opt in enumerate(lowerCamelCase		):
												snake_case__       :					str										=					opt.state_dict()
												snake_case__       :					Optional[Any]										=					f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
												snake_case__       :					str										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Optimizer state saved in {output_optimizer_file}'''		)
						# Scheduler states
						for i, scheduler in enumerate(lowerCamelCase		):
												snake_case__       :					int										=					scheduler.state_dict()
												snake_case__       :					List[Any]										=					f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
												snake_case__       :					int										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Scheduler state saved in {output_scheduler_file}'''		)
						# GradScaler state
						if scaler is not None:
												snake_case__       :					Tuple										=					scaler.state_dict()
												snake_case__       :					Tuple										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												torch.save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Gradient scaler state saved in {output_scaler_file}'''		)
						# Random number generator states
						snake_case__       :					Dict										=					{}
						snake_case__       :					Dict										=					f'''{RNG_STATE_NAME}_{process_index}.pkl'''
						snake_case__       :					Dict										=					random.getstate()
						snake_case__       :					List[str]										=					np.random.get_state()
						snake_case__       :					str										=					torch.get_rng_state()
						if is_xpu_available():
												snake_case__       :					List[Any]										=					torch.xpu.get_rng_state_all()
						else:
												snake_case__       :					Union[str, Any]										=					torch.cuda.get_rng_state_all()
						if is_tpu_available():
												snake_case__       :					int										=					xm.get_rng_state()
						snake_case__       :					int										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
						torch.save(lowerCamelCase      ,			lowerCamelCase		)
						logger.info(f'''Random states saved in {output_states_file}'''		)
						return output_dir

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase=None      ,			lowerCamelCase=None      ,			**lowerCamelCase      ,			):
						'''simple docstring'''


						if map_location not in [None, "cpu", "on_device"]:
												raise TypeError(
												    '''Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'''		)
						if map_location is None:
												snake_case__       :					str										=					'''cpu'''
						elif map_location == "on_device":
												snake_case__       :					List[Any]										=					PartialState().device
						# Model states
						for i, model in enumerate(lowerCamelCase		):
												snake_case__       :					Optional[int]										=					f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
												snake_case__       :					int										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												models[i].load_state_dict(torch.load(lowerCamelCase      ,			map_location=lowerCamelCase		)      ,			**lowerCamelCase		)
						logger.info('''All model weights loaded successfully'''		)

						# Optimizer states
						for i, opt in enumerate(lowerCamelCase		):
												snake_case__       :					Any										=					f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
												snake_case__       :					Optional[Any]										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												snake_case__       :					Any										=					torch.load(lowerCamelCase      ,			map_location=lowerCamelCase		)
												optimizers[i].load_state_dict(lowerCamelCase		)
						logger.info('''All optimizer states loaded successfully'''		)

						# Scheduler states
						for i, scheduler in enumerate(lowerCamelCase		):
												snake_case__       :					Optional[int]										=					f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
												snake_case__       :					Optional[Any]										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												scheduler.load_state_dict(torch.load(lowerCamelCase		)		)
						logger.info('''All scheduler states loaded successfully'''		)

						# GradScaler state
						if scaler is not None:
												snake_case__       :					Optional[Any]										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												scaler.load_state_dict(torch.load(lowerCamelCase		)		)
												logger.info('''GradScaler state loaded successfully'''		)

						# Random states
						try:
												snake_case__       :					Optional[int]										=					torch.load(os.path.join(lowerCamelCase      ,			f'''{RNG_STATE_NAME}_{process_index}.pkl'''		)		)
												random.setstate(states['''random_state''']		)
												np.random.set_state(states['''numpy_random_seed''']		)
												torch.set_rng_state(states['''torch_manual_seed''']		)
												if is_xpu_available():
																		torch.xpu.set_rng_state_all(states['''torch_xpu_manual_seed''']		)
												else:
																		torch.cuda.set_rng_state_all(states['''torch_cuda_manual_seed''']		)
												if is_tpu_available():
																		xm.set_rng_state(states['''xm_seed''']		)
												logger.info('''All random states loaded successfully'''		)
						except Exception:
												logger.info('''Could not load random states'''		)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase = 0		):
						'''simple docstring'''


						snake_case__       :					Union[str, Any]										=					Path(lowerCamelCase		) / f'''custom_checkpoint_{index}.pkl'''
						logger.info(f'''Saving the state of {get_pretty_name(lowerCamelCase		)} to {save_location}'''		)
						torch.save(obj.state_dict()      ,			lowerCamelCase		)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase = 0		):
						'''simple docstring'''


						snake_case__       :					Any										=					f'''{path}/custom_checkpoint_{index}.pkl'''
						logger.info(f'''Loading the state of {get_pretty_name(lowerCamelCase		)} from {load_location}'''		)
						obj.load_state_dict(torch.load(lowerCamelCase      ,			map_location='''cpu'''		)		)


