





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCAmelCase							:					Optional[Any]		    =		logging.get_logger(__name__)

_lowerCAmelCase							:					Any		    =		{
    '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/config.json''',
    '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/config.json''',
    '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/config.json''',
    '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/config.json''',
    '''bert-base-multilingual-uncased''': '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json''',
    '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json''',
    '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/config.json''',
    '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/config.json''',
    '''bert-large-uncased-whole-word-masking''': (
        '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json'''
    ),
    '''bert-large-cased-whole-word-masking''': (
        '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json'''
    ),
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
        '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json'''
    ),
    '''bert-large-cased-whole-word-masking-finetuned-squad''': (
        '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json'''
    ),
    '''bert-base-cased-finetuned-mrpc''': '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json''',
    '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json''',
    '''bert-base-german-dbmdz-uncased''': '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json''',
    '''cl-tohoku/bert-base-japanese''': '''https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json''',
    '''cl-tohoku/bert-base-japanese-whole-word-masking''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json'''
    ),
    '''cl-tohoku/bert-base-japanese-char''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json'''
    ),
    '''cl-tohoku/bert-base-japanese-char-whole-word-masking''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json'''
    ),
    '''TurkuNLP/bert-base-finnish-cased-v1''': (
        '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json'''
    ),
    '''TurkuNLP/bert-base-finnish-uncased-v1''': (
        '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json'''
    ),
    '''wietsedv/bert-base-dutch-cased''': '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json''',
    # See all BERT models at https://huggingface.co/models?filter=bert
}


class     __magic_name__     (							__A							):





     """simple docstring"""



     __UpperCamelCase    			=   'bert'



     def __init__(							self				:Union[str, Any]	,					snake_case				:str=30_522	,					snake_case				:List[str]=768	,					snake_case				:Any=12	,					snake_case				:Dict=12	,					snake_case				:int=3_072	,					snake_case				:Any="gelu"	,					snake_case				:str=0.1	,					snake_case				:Union[str, Any]=0.1	,					snake_case				:str=512	,					snake_case				:Union[str, Any]=2	,					snake_case				:Optional[int]=0.02	,					snake_case				:Tuple=1e-12	,					snake_case				:List[str]=0	,					snake_case				:List[str]="absolute"	,					snake_case				:Optional[Any]=True	,					snake_case				:Any=None	,					**snake_case				:int	,					):





        '''simple docstring'''




        super().__init__(pad_token_id=__UpperCAmelCase	,					**__UpperCAmelCase     )

        A_    : Optional[int]								=	vocab_size
        A_    : Optional[Any]								=	hidden_size
        A_    : Dict								=	num_hidden_layers
        A_    : List[Any]								=	num_attention_heads
        A_    : Dict								=	hidden_act
        A_    : Any								=	intermediate_size
        A_    : Dict								=	hidden_dropout_prob
        A_    : Optional[int]								=	attention_probs_dropout_prob
        A_    : Tuple								=	max_position_embeddings
        A_    : Union[str, Any]								=	type_vocab_size
        A_    : Dict								=	initializer_range
        A_    : Optional[Any]								=	layer_norm_eps
        A_    : List[Any]								=	position_embedding_type
        A_    : Union[str, Any]								=	use_cache
        A_    : List[str]								=	classifier_dropout




class     __magic_name__     (							__A							):





     """simple docstring"""



     @property
     def   SCREAMING_SNAKE_CASE						(							self				:List[Any]     ):





        '''simple docstring'''




        if self.task == "multiple-choice":
           A_    : List[Any]								=	{0: "batch", 1: "choice", 2: "sequence"}
        else:
           A_    : int								=	{0: "batch", 1: "sequence"}
        return OrderedDict(
            [
                ("input_ids", dynamic_axis),
                ("attention_mask", dynamic_axis),
                ("token_type_ids", dynamic_axis),
            ]     )

"""simple docstring"""




import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
				from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							if is_torch_version('<'				,			'2.0.0' ) or not hasattr(UpperCamelCase__				,			'_dynamo' ):
														return False
							return isinstance(UpperCamelCase__				,			torch._dynamo.eval_frame.OptimizedModule )






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__ = True ):






							"""simple docstring"""





							A__		   =							(torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

							A__		   =							is_compiled_module(UpperCamelCase__ )
							if is_compiled:
														A__		   =							model
														A__		   =							model._orig_mod

							if is_deepspeed_available():
														options += (DeepSpeedEngine,)

							while isinstance(UpperCamelCase__				,			UpperCamelCase__ ):
														A__		   =							model.module

							if not keep_fpaa_wrapper:
														A__		   =							getattr(UpperCamelCase__				,			'forward' )
														A__		   =							model.__dict__.pop('_original_forward'				,			UpperCamelCase__ )
														if original_forward is not None:
																					while hasattr(UpperCamelCase__				,			'__wrapped__' ):
																												A__		   =							forward.__wrapped__
																												if forward == original_forward:
																																			break
																					A__		   =							forward
														if getattr(UpperCamelCase__				,			'_converted_to_transformer_engine'				,			UpperCamelCase__ ):
																					convert_model(UpperCamelCase__				,			to_transformer_engine=UpperCamelCase__ )

							if is_compiled:
														A__		   =							model
														A__		   =							compiled_model

							return model






def      UpperCAmelCase    (		):






							"""simple docstring"""





							PartialState().wait_for_everyone()






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							if PartialState().distributed_type == DistributedType.TPU:
														xm.save(UpperCamelCase__				,			UpperCamelCase__ )
							elif PartialState().local_process_index == 0:
														torch.save(UpperCamelCase__				,			UpperCamelCase__ )






@contextmanager
def      UpperCAmelCase    (		**UpperCamelCase__ ):






							"""simple docstring"""





							for key, value in kwargs.items():
														A__		   =							str(UpperCamelCase__ )

							yield

							for key in kwargs:
														if key.upper() in os.environ:
																					del os.environ[key.upper()]






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							if not hasattr(UpperCamelCase__				,			'__qualname__' ) and not hasattr(UpperCamelCase__				,			'__name__' ):
														A__		   =							getattr(UpperCamelCase__				,			'__class__'				,			UpperCamelCase__ )
							if hasattr(UpperCamelCase__				,			'__qualname__' ):
														return obj.__qualname__
							if hasattr(UpperCamelCase__				,			'__name__' ):
														return obj.__name__
							return str(UpperCamelCase__ )






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							for key, value in source.items():
														if isinstance(UpperCamelCase__				,			UpperCamelCase__ ):
																					A__		   =							destination.setdefault(UpperCamelCase__				,			{} )
																					merge_dicts(UpperCamelCase__				,			UpperCamelCase__ )
														else:
																					A__		   =							value

							return destination






def      UpperCAmelCase    (		UpperCamelCase__ = None ):






							"""simple docstring"""





							if port is None:
														A__		   =							29_500
							with socket.socket(socket.AF_INET				,			socket.SOCK_STREAM ) as s:
														return s.connect_ex(('localhost', port) ) == 0





