code: 
  
  
  
  
  
  import torch
  
  from accelerate import Accelerator, DistributedType
  
  
  class 		_SCREAMING_SNAKE_CASE    :
  
  
       def __enter__(     self						):
         if self.enabled:
           _UpperCAmelCase		:       int       =    self.model.no_sync()
           self.model_sync_obj.__enter__()
  
         return self
  
  
       def __exit__(     self       ,				A_       ,				A_       ,				A_						):
         if self.enabled:
           # Average all models on exit
           self._sync_and_avg_model_params()
           self.model_sync_obj.__exit__(__UpperCamelCase       ,				__UpperCamelCase       ,				__UpperCamelCase						)
  
  
       def __init__(     self       ,				A_       ,				A_       ,				A_       ,				A_ = True						):
         if accelerator.distributed_type not in [
             DistributedType.NO,
             DistributedType.MULTI_CPU,
             DistributedType.MULTI_GPU,
         ]:
           raise NotImplementedError("""LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"""						)
         _UpperCAmelCase		:       str       =    enabled and accelerator.distributed_type != DistributedType.NO
         _UpperCAmelCase		:       Optional[Any]       =    0
         if self.enabled:
           _UpperCAmelCase		:       Optional[Any]       =    accelerator
           _UpperCAmelCase		:       Dict       =    model
           _UpperCAmelCase		:       Any       =    local_sgd_steps
  
  
       def 					__snake_case(     self						):
         self.num_steps += 1
         if not self.enabled:
           return
  
         if self.num_steps % self.local_sgd_steps == 0:
           self._sync_and_avg_model_params()
  
  
  
  
       def 					__snake_case(     self						):
         self.accelerator.wait_for_everyone()
         with self.accelerator.autocast():
           for param in self.model.parameters():
             _UpperCAmelCase		:       List[str]       =    self.accelerator.reduce(param.data       ,				reduction="""mean"""						)
  
  
  
  
  
  
style_context: 
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available
  
  
  SCREAMING_SNAKE_CASE__				:    List[str]         =    {
      'configuration_flaubert': ['FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FlaubertConfig', 'FlaubertOnnxConfig'],
      'tokenization_flaubert': ['FlaubertTokenizer'],
  }
  
  try:
   if not is_torch_available():
    raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
   pass
  else:
   SCREAMING_SNAKE_CASE__				:    int         =    [
       'FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'FlaubertForMultipleChoice',
       'FlaubertForQuestionAnswering',
       'FlaubertForQuestionAnsweringSimple',
       'FlaubertForSequenceClassification',
       'FlaubertForTokenClassification',
       'FlaubertModel',
       'FlaubertWithLMHeadModel',
       'FlaubertPreTrainedModel',
   ]
  
  try:
   if not is_tf_available():
    raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
   pass
  else:
   SCREAMING_SNAKE_CASE__				:    Dict         =    [
       'TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'TFFlaubertForMultipleChoice',
       'TFFlaubertForQuestionAnsweringSimple',
       'TFFlaubertForSequenceClassification',
       'TFFlaubertForTokenClassification',
       'TFFlaubertModel',
       'TFFlaubertPreTrainedModel',
       'TFFlaubertWithLMHeadModel',
   ]
  
  
  if TYPE_CHECKING:
   from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig, FlaubertOnnxConfig
   from .tokenization_flaubert import FlaubertTokenizer
  
   try:
    if not is_torch_available():
     raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
    pass
   else:
    from .modeling_flaubert import (
        FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        FlaubertForMultipleChoice,
        FlaubertForQuestionAnswering,
        FlaubertForQuestionAnsweringSimple,
        FlaubertForSequenceClassification,
        FlaubertForTokenClassification,
        FlaubertModel,
        FlaubertPreTrainedModel,
        FlaubertWithLMHeadModel,
    )
  
   try:
    if not is_tf_available():
     raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
    pass
   else:
    from .modeling_tf_flaubert import (
        TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFFlaubertForMultipleChoice,
        TFFlaubertForQuestionAnsweringSimple,
        TFFlaubertForSequenceClassification,
        TFFlaubertForTokenClassification,
        TFFlaubertModel,
        TFFlaubertPreTrainedModel,
        TFFlaubertWithLMHeadModel,
    )
  
  else:
   import sys
  
   SCREAMING_SNAKE_CASE__				:    List[Any]         =    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
  
  
  
  
  
  